//
// The Epoch Language Project
// EPOCHRUNTIME Runtime Library
//
// Just-in-time native code generation for Epoch
//
// The Epoch JIT engine is basically a thin translation layer that converts Epoch bytecode
// into LLVM bitcode instructions. Once the applicable conversions have been made, including
// relevant transformations of operations into SSA form (insofar as we can do that here),
// the LLVM bitcode is optimized and converted to native machine code on the fly. This occurs
// during the image load process prior to the execution of the Epoch entrypoint task.
//
// We rely extremely heavily on certain LLVM optimization passes to make efficient code out
// of our comparatively sloppy Epoch bytecode. There are also a large number of redundant
// operations and other expensive processing performed in the Epoch VM which can be converted
// to faster forms or even eliminated entirely post-LLVM optimizations. As such, it is the
// general case that we do not bother doing much optimization of Epoch bytecode. As it stands,
// Epoch bytecode is a high-level and expressive form which is easy to emit from the compiler.
// LLVM bitcode can be generated relatively easily from this form, and the resulting bitcode
// can be optimized into very efficient machine code without any additional effort from us.
//
// Low-level optimizations are almost entirely handled as well as can be expected by LLVM.
// For the time being, the primary means of improving native machine code generated by an
// Epoch program is to observe LLVM's behavior when handed the sorts of idioms that Epoch
// bytecode tends to generate. We can tweak the optimization pass order and sequence at the
// JIT layer, which gives us a powerful way to ensure that LLVM can cleanly produce machine
// code from our inputs, and yet does not necessitate writing custom passes or doing any
// specific magic at a higher level.
//
// That said, the potential for adding high-level algorithmic type optimizations to the
// Epoch compiler is considerable. Given the type system richness of Epoch (especially as
// compared to LLVM) and the general high availability of metadata in the Epoch bytecode
// stream, it should be practical to annotate the LLVM bitcode in certain ways which further
// improve that layer's ability to generate optimal machine code.
//

#include "pch.h"

#include "Bytecode/Instructions.h"

#include "Libraries/Library.h"

#include "Runtime/Runtime.h"
#include "Runtime/Marshaling.h"
#include "Runtime/GlobalContext.h"

#include "JIT/GarbageCollection.h"

#include "Metadata/ScopeDescription.h"
#include "Metadata/TypeInfo.h"

#include "Utility/Strings.h"

#include <llvm/IR/Intrinsics.h>
#include <llvm/Support/CrashRecoveryContext.h>
#include <llvm/Transforms/Vectorize.h>
#include <llvm/ADT/Triple.h>

#pragma warning(push)
#pragma warning(disable: 4146)
#pragma warning(disable: 4100)
#include <llvm/ADT/Statistic.h>
#pragma warning(pop)

#include <sstream>
#include <map>
#include <stack>
#include <iostream>


// Enable this to turn on profiling (slow!)
//#define PROFILING_HOOKS


// We heavily use this namespace so may as well import it
using namespace llvm;

//
// Internal implementation helpers
//
namespace JIT
{
	namespace impl
	{

		ExecutionEngine* LazyEngine = NULL;

		//
		// The JIT operations maintain a lot of state and other data
		// which is useful to sort and centralize in some manner. We
		// collect a large part of this data in the following struct
		// in order to share state between assorted JIT helper code.
		//
		struct LLVMData
		{
			explicit LLVMData(Module* module);

			LLVMContext& Context;

			Module* CurrentModule;

			Type* TypeIDType;
			Type* BufferHandleType;
			Type* StructureHandleType;
			Type* NothingType;
			
			std::map<JITFunctionID, Function*> BuiltInFunctions;

			std::map<std::string, Function*> GeneratedFunctions;
			std::map<std::string, Function*> GeneratedNativeTypeMatchers;
			std::map<StringHandle, Function*> GeneratedNameToFunctionMap;
			std::map<Function*, StringHandle> GeneratedFunctionToNameMap;

			std::map<Metadata::EpochTypeID, StructType*> SumTypeCache;

			std::map<size_t, Value*> GlobalVariableMap;
			std::map<const Value*, Metadata::EpochTypeID> GlobalVariableTypes;
			std::map<size_t, size_t> GlobalVariableOffsetToIndexMap;
			std::map<StringHandle, size_t> GlobalVariableNameToIndexMap;

			Function* EntryPoint;
			Function* GlobalInit;

		// Non-copyable
		private:
			LLVMData(const LLVMData&);
			LLVMData& operator = (const LLVMData&);
		};

		//
		// Yet more JIT state is maintained on a per-function level.
		// This helper class holds that state during the JIT process
		// for each individual function for which we generate native
		// code. It also provides organized implementations of logic
		// for handling various Epoch bytecode instructions.
		//
		class FunctionJITHelper
		{
		// Construction
		public:
			explicit FunctionJITHelper(NativeCodeGenerator& generator);

		// Non-copyable
		private:
			FunctionJITHelper(const FunctionJITHelper&);
			FunctionJITHelper& operator = (const FunctionJITHelper&);

		// Function JIT interface
		public:
			void DoFunction(size_t beginoffset, size_t endoffset, StringHandle alias);
			void DoTypeMatchedFunction(size_t beginoffset, size_t endoffset, StringHandle alias, const std::vector<Value*>& params, Function* wrap);

		// Global init JIT interface
		public:
			void DoGlobalInit(size_t beginoffset, StringHandle alias);

		// Instruction handlers
		private:
			void BeginEntity(size_t& offset);
			void EndEntity(size_t& offset);

			void BeginChain(size_t& offset);
			void EndChain(size_t& offset);

			void Read(size_t& offset);
			void ReadStackLocal(size_t& offset);
			void ReadParameter(size_t& offset);

			void BindReference(size_t& offset);
			void ReadRef(size_t& offset);
			void ReadRefAnnotated(size_t& offset);

			void Assign(size_t& offset);
			void AssignSumType(size_t& offset);
			void ConstructSumType(size_t& offset);

			void SetRetValue(size_t& offset);
			void Return(size_t& offset);
			void Halt(size_t& offset);

			void Push(size_t& offset);
			void Pop(size_t& offset);

			void AllocStructure(size_t& offset);
			void CopyToStructure(size_t& offset);
			void CopyStructure(size_t& offset);
			void BindMemberRef(size_t& offset);

			void Invoke(size_t& offset);
			void InvokeOffset(size_t& offset);
			void InvokeNative(size_t& offset);
			void InvokeIndirect(size_t& offset);

			void TypeMatch(size_t& offset);
			void PatternMatch(size_t& offset);

			void CopyBuffer(size_t& offset);

		// Additional helpers
		private:
			AllocaInst* CreateAllocaInternal(Type* type, bool forceintofirstblock = false);

		// Internal tracking
		private:
			NativeCodeGenerator& Generator;
			const Bytecode::Instruction* Bytecode;

			IRBuilder<>& Builder;
			LLVMContext& Context;

			JITContext LibJITContext;

			BasicBlock* NativeMatchBlock;
			BasicBlock* PatternMatchBlock;

			size_t BeginOffset;
			size_t EndOffset;

			unsigned NumParameters;
			unsigned NumParameterBytes;
			unsigned NumReturns;

			unsigned AllocCount;
			unsigned ChainDepth;

			Metadata::EpochTypeID HackStructType;		// TODO - clean this up

			std::stack<Metadata::EpochTypeID> TypeAnnotations;
			std::map<size_t, size_t> LocalOffsetToIndexMap;
			std::map<size_t, size_t> ParameterOffsetToIndexMap;

			typedef void (FunctionJITHelper::* InstructionJITHelper)(size_t& offset);
			std::map<Bytecode::Instruction, InstructionJITHelper> InstructionJITHelpers;

			bool CompilingTypeMatchedFunction;
			const std::vector<Value*>* TypeMatchedParams;
			Function* WrapperFunction;

			friend class NativeCodeGenerator;
		};

		//
		// Construct and initialize an LLVM data wrapper object
		//
		LLVMData::LLVMData(Module* module) :
			Context(getGlobalContext()),
			CurrentModule(module)
		{
			TypeIDType = Type::getInt32Ty(Context);
			BufferHandleType = Type::getInt32Ty(Context);
			StructureHandleType = Type::getInt8PtrTy(Context);
			NothingType = Type::getInt32Ty(Context);

			// Set up various interop functions
			if(!CurrentModule->getFunction("Epoch_Break"))
			{
				FunctionType* ftype = FunctionType::get(Type::getVoidTy(Context), false);
				BuiltInFunctions[JITFunc_Runtime_Break] = Function::Create(ftype, Function::ExternalLinkage, "Epoch_Break", CurrentModule);
			}
			else
				BuiltInFunctions[JITFunc_Runtime_Break] = CurrentModule->getFunction("Epoch_Break");

			if(!CurrentModule->getFunction("Epoch_Halt"))
			{
				FunctionType* ftype = FunctionType::get(Type::getVoidTy(Context), false);
				BuiltInFunctions[JITFunc_Runtime_Halt] = Function::Create(ftype, Function::ExternalLinkage, "Epoch_Halt", CurrentModule);
			}
			else
				BuiltInFunctions[JITFunc_Runtime_Halt] = CurrentModule->getFunction("Epoch_Halt");

			if(!CurrentModule->getFunction("Epoch_HaltExt"))
			{
				std::vector<Type*> args;
				args.push_back(Type::getInt32Ty(Context));
				FunctionType* ftype = FunctionType::get(Type::getVoidTy(Context), args, false);

				BuiltInFunctions[JITFunc_Runtime_HaltExt] = Function::Create(ftype, Function::ExternalLinkage, "Epoch_HaltExt", CurrentModule);
			}
			else
				BuiltInFunctions[JITFunc_Runtime_HaltExt] = CurrentModule->getFunction("Epoch_HaltExt");

			if(!CurrentModule->getFunction("Epoch_ProfileEnter"))
			{
				std::vector<Type*> args;
				args.push_back(Type::getInt32Ty(Context));
				FunctionType* ftype = FunctionType::get(Type::getVoidTy(Context), args, false);

				BuiltInFunctions[JITFunc_Profile_Enter] = Function::Create(ftype, Function::ExternalLinkage, "Epoch_ProfileEnter", CurrentModule);
			}
			else
				BuiltInFunctions[JITFunc_Profile_Enter] = CurrentModule->getFunction("Epoch_ProfileEnter");

			if(!CurrentModule->getFunction("Epoch_ProfileExit"))
			{
				std::vector<Type*> args;
				args.push_back(Type::getInt32Ty(Context));
				FunctionType* ftype = FunctionType::get(Type::getVoidTy(Context), args, false);

				BuiltInFunctions[JITFunc_Profile_Exit] = Function::Create(ftype, Function::ExternalLinkage, "Epoch_ProfileExit", CurrentModule);
			}
			else
				BuiltInFunctions[JITFunc_Profile_Exit] = CurrentModule->getFunction("Epoch_ProfileExit");

			if(!CurrentModule->getFunction("Epoch_ProfileDump"))
			{
				FunctionType* ftype = FunctionType::get(Type::getVoidTy(Context), false);
				BuiltInFunctions[JITFunc_Profile_Dump] = Function::Create(ftype, Function::ExternalLinkage, "Epoch_ProfileDump", CurrentModule);
			}
			else
				BuiltInFunctions[JITFunc_Profile_Dump] = CurrentModule->getFunction("Epoch_ProfileDump");

			if(!CurrentModule->getFunction("Epoch_AllocStruct"))
			{
				std::vector<Type*> args;
				args.push_back(TypeIDType);
				FunctionType* ftype = FunctionType::get(StructureHandleType, args, false);

				BuiltInFunctions[JITFunc_Runtime_AllocStruct] = Function::Create(ftype, Function::ExternalLinkage, "Epoch_AllocStruct", CurrentModule);
			}
			else
				BuiltInFunctions[JITFunc_Runtime_AllocStruct] = CurrentModule->getFunction("Epoch_AllocStruct");


			if(!CurrentModule->getFunction("Epoch_CopyStruct"))
			{
				std::vector<Type*> args;
				args.push_back(StructureHandleType);
				FunctionType* ftype = FunctionType::get(StructureHandleType, args, false);

				BuiltInFunctions[JITFunc_Runtime_CopyStruct] = Function::Create(ftype, Function::ExternalLinkage, "Epoch_CopyStruct", CurrentModule);
			}
			else
				BuiltInFunctions[JITFunc_Runtime_CopyStruct] = CurrentModule->getFunction("Epoch_CopyStruct");

			if(!CurrentModule->getFunction("Epoch_GetBuffer"))
			{
				std::vector<Type*> args;
				args.push_back(BufferHandleType);
				FunctionType* ftype = FunctionType::get(Type::getInt8PtrTy(Context), args, false);

				BuiltInFunctions[JITFunc_Runtime_GetBuffer] = Function::Create(ftype, Function::ExternalLinkage, "Epoch_GetBuffer", CurrentModule);
				BuiltInFunctions[JITFunc_Runtime_GetBuffer]->addFnAttr(llvm::Attribute::ReadNone);
			}
			else
				BuiltInFunctions[JITFunc_Runtime_GetBuffer] = CurrentModule->getFunction("Epoch_GetBuffer");


			if(!CurrentModule->getFunction("Epoch_GetBufferByPtr"))
			{
				std::vector<Type*> args;
				args.push_back(Type::getInt8PtrTy(Context));
				FunctionType* ftype = FunctionType::get(BufferHandleType, args, false);

				BuiltInFunctions[JITFunc_Runtime_PtrToBufferHandle] = Function::Create(ftype, Function::ExternalLinkage, "Epoch_GetBufferByPtr", CurrentModule);
			}
			else
				BuiltInFunctions[JITFunc_Runtime_PtrToBufferHandle] = CurrentModule->getFunction("Epoch_GetBufferByPtr");

			if(!CurrentModule->getFunction("Epoch_GetString"))
			{
				std::vector<Type*> args;
				args.push_back(Type::getInt32Ty(Context));
				FunctionType* ftype = FunctionType::get(Type::getInt8PtrTy(Context), args, false);

				BuiltInFunctions[JITFunc_Runtime_GetString] = Function::Create(ftype, Function::ExternalLinkage, "Epoch_GetString", CurrentModule);
				BuiltInFunctions[JITFunc_Runtime_GetString]->addFnAttr(llvm::Attribute::ReadNone);
			}
			else
				BuiltInFunctions[JITFunc_Runtime_GetString] = CurrentModule->getFunction("Epoch_GetString");

			if(!CurrentModule->getFunction("CRT_StrCmp"))
			{
				std::vector<Type*> args;
				args.push_back(Type::getInt8PtrTy(Context));
				args.push_back(Type::getInt8PtrTy(Context));
				FunctionType* ftype = FunctionType::get(Type::getInt32Ty(Context), args, false);

				BuiltInFunctions[JITFunc_CRuntime_StrCmp] = Function::Create(ftype, Function::ExternalLinkage, "wcscmp", CurrentModule);
			}
			else
				BuiltInFunctions[JITFunc_CRuntime_StrCmp] = CurrentModule->getFunction("CRT_StrCmp");

			if(!CurrentModule->getFunction("Epoch_PoolString"))
			{
				std::vector<Type*> args;
				args.push_back(Type::getInt8PtrTy(Context));
				FunctionType* ftype = FunctionType::get(Type::getInt32Ty(Context), args, false);

				BuiltInFunctions[JITFunc_Runtime_PoolString] = Function::Create(ftype, Function::ExternalLinkage, "Epoch_PoolString", CurrentModule);
			}
			else
				BuiltInFunctions[JITFunc_Runtime_PoolString] = CurrentModule->getFunction("Epoch_PoolString");

			if(!CurrentModule->getFunction("Epoch_AllocBuffer"))
			{
				std::vector<Type*> args;
				args.push_back(Type::getInt32Ty(Context));
				FunctionType* ftype = FunctionType::get(Type::getInt32Ty(Context), args, false);

				BuiltInFunctions[JITFunc_Runtime_AllocBuffer] = Function::Create(ftype, Function::ExternalLinkage, "Epoch_AllocBuffer", CurrentModule);
			}
			else
				BuiltInFunctions[JITFunc_Runtime_AllocBuffer] = CurrentModule->getFunction("Epoch_AllocBuffer");

			if(!CurrentModule->getFunction("Epoch_CopyBuffer"))
			{
				std::vector<Type*> args;
				args.push_back(Type::getInt32Ty(Context));
				FunctionType* ftype = FunctionType::get(Type::getInt32Ty(Context), args, false);

				BuiltInFunctions[JITFunc_Runtime_CopyBuffer] = Function::Create(ftype, Function::ExternalLinkage, "Epoch_CopyBuffer", CurrentModule);
			}
			else
				BuiltInFunctions[JITFunc_Runtime_CopyBuffer] = CurrentModule->getFunction("Epoch_CopyBuffer");

			if(!CurrentModule->getFunction("TriggerGarbageCollection"))
			{
				std::vector<Type*> args;
				FunctionType* ftype = FunctionType::get(Type::getVoidTy(Context), args, false);

				BuiltInFunctions[JITFunc_Runtime_TriggerGC] = Function::Create(ftype, Function::ExternalLinkage, "TriggerGarbageCollection", CurrentModule);
				Runtime::GetThreadContext()->TriggerGCFunc = BuiltInFunctions[JITFunc_Runtime_TriggerGC];
			}
			else
				BuiltInFunctions[JITFunc_Runtime_TriggerGC] = CurrentModule->getFunction("TriggerGarbageCollection");

			if(!CurrentModule->getFunction("GCBookmark"))
			{
				std::vector<Type*> args;
				FunctionType* ftype = FunctionType::get(Type::getVoidTy(Context), args, false);

				BuiltInFunctions[JITFunc_Runtime_GCBookmark] = Function::Create(ftype, Function::ExternalLinkage, "GCBookmark", CurrentModule);
			}
			else
				BuiltInFunctions[JITFunc_Runtime_GCBookmark] = CurrentModule->getFunction("GCBookmark");

			if(!CurrentModule->getFunction("GCUnbookmark"))
			{
				std::vector<Type*> args;
				FunctionType* ftype = FunctionType::get(Type::getVoidTy(Context), args, false);

				BuiltInFunctions[JITFunc_Runtime_GCUnbookmark] = Function::Create(ftype, Function::ExternalLinkage, "GCUnbookmark", CurrentModule);
			}
			else
				BuiltInFunctions[JITFunc_Runtime_GCUnbookmark] = CurrentModule->getFunction("GCUnbookmark");

			if(!CurrentModule->getFunction("MarshalConvertStructure"))
			{
				std::vector<Type*> args;
				args.push_back(Type::getInt8PtrTy(Context));
				FunctionType* ftype = FunctionType::get(Type::getInt8PtrTy(Context), args, false);

				BuiltInFunctions[JITFunc_Marshal_ConvertStructure] = Function::Create(ftype, Function::ExternalLinkage, "MarshalConvertStructure", CurrentModule);
			}
			else
				BuiltInFunctions[JITFunc_Marshal_ConvertStructure] = CurrentModule->getFunction("MarshalConvertStructure");

			if(!CurrentModule->getFunction("MarshalFixupStructure"))
			{
				std::vector<Type*> args;
				args.push_back(Type::getInt8PtrTy(Context));
				args.push_back(Type::getInt8PtrTy(Context));
				FunctionType* ftype = FunctionType::get(Type::getVoidTy(Context), args, false);

				BuiltInFunctions[JITFunc_Marshal_FixupStructure] = Function::Create(ftype, Function::ExternalLinkage, "MarshalFixupStructure", CurrentModule);
			}
			else
				BuiltInFunctions[JITFunc_Marshal_FixupStructure] = CurrentModule->getFunction("MarshalFixupStructure");

			if(!CurrentModule->getFunction("MarshalCleanup"))
			{
				std::vector<Type*> args;
				args.push_back(Type::getInt8PtrTy(Context));
				FunctionType* ftype = FunctionType::get(Type::getVoidTy(Context), args, false);

				BuiltInFunctions[JITFunc_Marshal_Cleanup] = Function::Create(ftype, Function::ExternalLinkage, "MarshalCleanup", CurrentModule);
			}
			else
				BuiltInFunctions[JITFunc_Marshal_Cleanup] = CurrentModule->getFunction("MarshalCleanup");

			if(!CurrentModule->getFunction("MarshalGenCallback"))
			{
				std::vector<Type*> args;
				args.push_back(Type::getInt8PtrTy(Context));
				FunctionType* ftype = FunctionType::get(Type::getInt8PtrTy(Context), args, false);

				BuiltInFunctions[JITFunc_Marshal_GenCallback] = Function::Create(ftype, Function::ExternalLinkage, "MarshalGenCallback", CurrentModule);
			}
			else
				BuiltInFunctions[JITFunc_Marshal_GenCallback] = CurrentModule->getFunction("MarshalGenCallback");

			// Set up intrinsics
			if(!CurrentModule->getFunction("llvm.sqrt.f32"))
			{
				std::vector<Type*> args;
				args.push_back(Type::getFloatTy(Context));
				FunctionType* ftype = FunctionType::get(Type::getFloatTy(Context), args, false);
				BuiltInFunctions[JITFunc_Intrinsic_Sqrt] = Function::Create(ftype, Function::ExternalLinkage, "llvm.sqrt.f32", CurrentModule);
			}
			else
				BuiltInFunctions[JITFunc_Intrinsic_Sqrt] = CurrentModule->getFunction("llvm.sqrt.f32");

			if(!CurrentModule->getFunction("llvm.gcroot"))
			{
				std::vector<Type*> args;
				args.push_back(Type::getInt8PtrTy(Context)->getPointerTo());
				args.push_back(Type::getInt8PtrTy(Context));
				FunctionType* ftype = FunctionType::get(Type::getVoidTy(Context), args, false);
				BuiltInFunctions[JITFunc_Intrinsic_GCRoot] = Function::Create(ftype, Function::ExternalLinkage, "llvm.gcroot", CurrentModule);
			}
			else
				BuiltInFunctions[JITFunc_Intrinsic_GCRoot] = CurrentModule->getFunction("llvm.gcroot");

			if(!CurrentModule->getFunction("llvm.va_start"))
			{
				std::vector<Type*> args;
				args.push_back(Type::getInt8PtrTy(Context));
				FunctionType* ftype = FunctionType::get(Type::getVoidTy(Context), args, false);
				BuiltInFunctions[JITFunc_Intrinsic_VAStart] = Function::Create(ftype, Function::ExternalLinkage, "llvm.va_start", CurrentModule);
			}
			else
				BuiltInFunctions[JITFunc_Intrinsic_VAStart] = CurrentModule->getFunction("llvm.va_start");

			if(!CurrentModule->getFunction("llvm.va_end"))
			{
				std::vector<Type*> args;
				args.push_back(Type::getInt8PtrTy(Context));
				FunctionType* ftype = FunctionType::get(Type::getVoidTy(Context), args, false);
				BuiltInFunctions[JITFunc_Intrinsic_VAEnd] = Function::Create(ftype, Function::ExternalLinkage, "llvm.va_end", CurrentModule);
			}
			else
				BuiltInFunctions[JITFunc_Intrinsic_VAEnd] = CurrentModule->getFunction("llvm.va_end");
		}

	}
}


//
// Additional internal helpers
//
namespace
{

	template <typename T>
	T Fetch(const Bytecode::Instruction* bytecode, size_t& InstructionOffset)
	{
		const T* data = reinterpret_cast<const T*>(&bytecode[InstructionOffset]);
		InstructionOffset += sizeof(T);
		return static_cast<T>(*data);
	}
	
}


// Namespace shortcuts
using namespace JIT;
using namespace JIT::impl;


//
// Construct and initialize a native code generation wrapper
//
NativeCodeGenerator::NativeCodeGenerator(Runtime::ExecutionContext& execcontext, const Bytecode::Instruction* bytecode)
	: ExecContext(execcontext),
	  Bytecode(bytecode),
	  Data(new LLVMData(new Module("EpochJIT", getGlobalContext()))),
	  Builder(Data->Context)
{
	InitializeNativeTarget();
}

//
// Destruct and clean up a native code generation wrapper
//
NativeCodeGenerator::~NativeCodeGenerator()
{
	delete Data;

	if(ExecContext.JITExecutionEngine)
		reinterpret_cast<ExecutionEngine*>(ExecContext.JITExecutionEngine)->UnregisterJITEventListener(this);
}


//
// Retrieve (or create if necessary) a generated function
//
Function* NativeCodeGenerator::GetGeneratedFunction(StringHandle funcname, size_t beginoffset)
{
	std::ostringstream name;
	name << "JIT_" << narrow(ExecContext.GetPooledString(funcname)) << "_" << beginoffset;

	Function* targetinnerfunc = NULL;
	FunctionType* targetinnerfunctype = GetLLVMFunctionType(funcname);
	if(!Data->GeneratedFunctions[name.str()])
	{
		targetinnerfunc = Function::Create(targetinnerfunctype, Function::ExternalLinkage, name.str().c_str(), Data->CurrentModule);
		Data->GeneratedFunctions[name.str()] = targetinnerfunc;
		Data->GeneratedFunctionToNameMap[targetinnerfunc] = funcname;

		if((ExecContext.AutoGeneratedConstructors.find(funcname) == ExecContext.AutoGeneratedConstructors.end())
		&& (!Runtime::IsMarshaledExternalFunction(funcname))
		&& (ExecContext.SuppressGC.find(funcname) == ExecContext.SuppressGC.end())
		  )
		{
			targetinnerfunc->setGC("EpochGC");
		}
	}
	else
		targetinnerfunc = Data->GeneratedFunctions[name.str()];

	return targetinnerfunc;
}

//
// Retrieve (or create if necessary) a generated type matcher
//
Function* NativeCodeGenerator::GetGeneratedTypeMatcher(StringHandle funcname, size_t beginoffset)
{
	std::ostringstream matchername;
	matchername << "JITMatcher_" << beginoffset << "_" << narrow(ExecContext.GetPooledString(funcname));

	Function* nativetypematcher = Data->GeneratedNativeTypeMatchers[matchername.str()];
	if(!nativetypematcher)
	{
		std::vector<Type*> matchargtypes;
		for(size_t i = 0; i < ExecContext.TypeMatcherParamCount.find(funcname)->second; ++i)
		{
			matchargtypes.push_back(Data->TypeIDType);					// type annotation
			matchargtypes.push_back(Type::getInt8PtrTy(Data->Context));		// pointer to payload
		}

		Type* retty = Type::getVoidTy(Data->Context);
		if(ExecContext.TypeMatcherRetType.find(funcname)->second != Metadata::EpochType_Error)
			retty = GetLLVMType(ExecContext.TypeMatcherRetType.find(funcname)->second);
		FunctionType* matchfunctype = FunctionType::get(retty, matchargtypes, false);

		nativetypematcher = Function::Create(matchfunctype, Function::ExternalLinkage, matchername.str().c_str(), Data->CurrentModule);
		
		//nativetypematcher->setGC("EpochGC");

		Data->GeneratedNativeTypeMatchers[matchername.str()] = nativetypematcher;
	}

	return nativetypematcher;
}

//
// Retrieve (or create if necessary) a generated pattern matcher
//
Function* NativeCodeGenerator::GetGeneratedPatternMatcher(StringHandle funcname, size_t beginoffset)
{
	std::ostringstream matchername;
	matchername << "JITMatcher_" << beginoffset << "_" << funcname;

	Function* nativematcher = Data->GeneratedNativeTypeMatchers[matchername.str()];
	if(!nativematcher)
	{
		StringHandle hint = ExecContext.PatternMatcherDispatchHint.find(funcname)->second;
		FunctionType* matchfunctype = GetLLVMFunctionType(hint);

		nativematcher = Function::Create(matchfunctype, Function::ExternalLinkage, matchername.str().c_str(), Data->CurrentModule);

		//nativematcher->setGC("EpochGC");

		Data->GeneratedNativeTypeMatchers[matchername.str()] = nativematcher;
	}

	return nativematcher;
}

//
// Retrieve (or create if necessary) a generated global initializer
//
Function* NativeCodeGenerator::GetGeneratedGlobalInit(StringHandle entityname)
{
	std::ostringstream name;
	name << "GlobalEntry_" << entityname;

	Function* targetinnerfunc = NULL;
	FunctionType* targetinnerfunctype = FunctionType::get(Type::getVoidTy(Data->Context), false);
	if(!Data->GeneratedFunctions[name.str()])
	{
		targetinnerfunc = Function::Create(targetinnerfunctype, Function::ExternalLinkage, name.str().c_str(), Data->CurrentModule);
		targetinnerfunc->setGC("EpochGC");
		Data->GeneratedFunctions[name.str()] = targetinnerfunc;
		Data->GeneratedFunctionToNameMap[targetinnerfunc] = entityname;
	}
	else
		targetinnerfunc = Data->GeneratedFunctions[name.str()];

	return targetinnerfunc;
}

//
// Map Epoch types to LLVM types
//
Type* NativeCodeGenerator::GetLLVMType(Metadata::EpochTypeID type, bool flatten)
{
	bool ref = false;
	if(Metadata::IsReferenceType(type))
	{
		ref = true;
		type = Metadata::MakeNonReferenceType(type);
	}

	Metadata::EpochTypeFamily family = Metadata::GetTypeFamily(type);
	switch(type)
	{
	case Metadata::EpochType_Void:
		return Type::getVoidTy(Data->Context);

	case Metadata::EpochType_Integer:
	case Metadata::EpochType_String:
	case Metadata::EpochType_Buffer:
		if(ref)
			return Type::getInt32PtrTy(Data->Context);
		else
			return Type::getInt32Ty(Data->Context);

	case Metadata::EpochType_Real:
		if(ref)
			return Type::getFloatPtrTy(Data->Context);
		else
			return Type::getFloatTy(Data->Context);

	case Metadata::EpochType_Boolean:
		if(ref)
			return Type::getInt1PtrTy(Data->Context);
		else
			return Type::getInt1Ty(Data->Context);

	case Metadata::EpochType_Integer16:
		if(ref)
			return Type::getInt16PtrTy(Data->Context);
		else
			return Type::getInt16Ty(Data->Context);

	case Metadata::EpochType_Identifier:
		// STUPID LAME HACK - we only use Identifier params for constructors, so assume this is an aggregate constructor param
		if(ref)
			return Data->StructureHandleType->getPointerTo();
		else
			return Data->StructureHandleType;

	default:
		if(family == Metadata::EpochTypeFamily_Function)
			return GetLLVMFunctionTypeFromEpochType(type)->getPointerTo();

		if(family == Metadata::EpochTypeFamily_SumType)
		{
			if(ref)
				return GetLLVMSumType(type, flatten)->getPointerTo();
			else
				return GetLLVMSumType(type, flatten);
		}

		if(Metadata::IsStructureType(type))
		{
			if(ref)
				return Data->StructureHandleType->getPointerTo();
			else 
				return Data->StructureHandleType;
		}

		throw NotImplementedException("Unsupported type for native code generation");
	}
}

//
// Map Epoch types to LLVM types suitable for C-ABI function invocation
//
Type* NativeCodeGenerator::GetExternalType(Metadata::EpochTypeID type)
{
	Metadata::EpochTypeFamily family = Metadata::GetTypeFamily(type);
	switch(type)
	{
	case Metadata::EpochType_Void:
		return Type::getVoidTy(Data->Context);

	case Metadata::EpochType_Integer:
		return Type::getInt32Ty(Data->Context);

	case Metadata::EpochType_String:
	case Metadata::EpochType_Buffer:
		return Type::getInt8PtrTy(Data->Context);

	case Metadata::EpochType_Real:
		return Type::getFloatTy(Data->Context);

	case Metadata::EpochType_Boolean:
		return Type::getInt32Ty(Data->Context);

	case Metadata::EpochType_Integer16:
		return Type::getInt16Ty(Data->Context);

	default:
		if(Metadata::IsStructureType(type))
			return Data->StructureHandleType;

		if(family == Metadata::EpochTypeFamily_Function)
			return Type::getInt8PtrTy(Data->Context);

		throw NotImplementedException("Unsupported type for marshaling to external functions");
	}
}


//
// Create tagged structures to hold Epoch unions in LLVM structs
//
// Since LLVM does not support unions, we need to fake this by creating
// a field of the largest necesary bit width and doing a lot of pointer
// casting when reading/writing the union payload.
//
Type* NativeCodeGenerator::GetLLVMSumType(Metadata::EpochTypeID type, bool flatten)
{
	StructType* taggedtype = Data->SumTypeCache[type];
	if(!taggedtype && !flatten)
	{
		const VariantDefinition& def = ExecContext.VariantDefinitions.find(type)->second;
		const std::set<Metadata::EpochTypeID>& types = def.GetBaseTypes();

		Type* rettype = NULL;
		size_t maxsize = 0;

		for(std::set<Metadata::EpochTypeID>::const_iterator iter = types.begin(); iter != types.end(); ++iter)
		{
			size_t size = Metadata::GetStorageSize(*iter);
			if(size > maxsize)
			{
				maxsize = size;
				rettype = GetLLVMType(*iter, true);
			}
		}

		if(flatten)
			return rettype;

		std::ostringstream name;
		name << "SumTypeTag_" << type;

		std::vector<Type*> elemtypes;
		elemtypes.push_back(Data->TypeIDType);
		elemtypes.push_back(rettype);
		taggedtype = StructType::create(elemtypes, name.str());
		Data->SumTypeCache[type] = taggedtype;
	}

	return taggedtype;
}

//
// Synthesize the LLVM function type signature for a given Epoch function
//
FunctionType* NativeCodeGenerator::GetLLVMFunctionType(StringHandle epochfunc)
{
	bool hassumtypeparam = false;
	bool isautogenconstructor = (ExecContext.AutoGeneratedConstructors.count(epochfunc) > 0);
	Type* rettype = Type::getVoidTy(Data->Context);

	if(ExecContext.PatternMatcherParamCount.find(epochfunc) != ExecContext.PatternMatcherParamCount.end())
		epochfunc = ExecContext.PatternMatcherDispatchHint[epochfunc];

	std::vector<Type*> args;

	const ScopeDescription& scope = ExecContext.GetScopeDescription(epochfunc);
	for(size_t i = 0; i < scope.GetVariableCount(); ++i)
	{
		if(scope.GetVariableOrigin(i) == VARIABLE_ORIGIN_PARAMETER)
		{
			Metadata::EpochTypeID vartype = scope.GetVariableTypeByIndex(i);

			if(vartype == Metadata::EpochType_Nothing)
			{
				args.push_back(Data->NothingType);
			}
			else
			{
				args.push_back(GetLLVMType(vartype));

				if(Metadata::GetTypeFamily(vartype) == Metadata::EpochTypeFamily_SumType)
					hassumtypeparam = true;
			}
		}
		else if(scope.GetVariableOrigin(i) == VARIABLE_ORIGIN_RETURN)
			rettype = GetLLVMType(scope.GetVariableTypeByIndex(i));
	}

	if(hassumtypeparam && isautogenconstructor)
		return FunctionType::get(rettype, std::vector<Type*>(), true);
	
	return FunctionType::get(rettype, args, false);
}

//
// Synthesize the LLVM function type signature for a given external C-ABI function
//
FunctionType* NativeCodeGenerator::GetExternalFunctionType(StringHandle epochfunc)
{
	Type* rettype = Type::getVoidTy(Data->Context);

	std::vector<Type*> args;

	const ScopeDescription& scope = ExecContext.GetScopeDescription(epochfunc);
	for(size_t i = 0; i < scope.GetVariableCount(); ++i)
	{
		if(scope.GetVariableOrigin(i) == VARIABLE_ORIGIN_PARAMETER)
		{
			Metadata::EpochTypeID vartype = scope.GetVariableTypeByIndex(i);
			Type* t = GetExternalType(Metadata::MakeNonReferenceType(vartype));
			if(Metadata::IsReferenceType(vartype) && !Metadata::IsStructureType(vartype) && Metadata::MakeNonReferenceType(vartype) != Metadata::EpochType_Buffer)
				t = t->getPointerTo();
			args.push_back(t);
		}
		else if(scope.GetVariableOrigin(i) == VARIABLE_ORIGIN_RETURN)
			rettype = GetExternalType(scope.GetVariableTypeByIndex(i));
	}

	return FunctionType::get(rettype, args, false);
}

//
// Synthesize the LLVM function type signature for a given library function
//
llvm::FunctionType* NativeCodeGenerator::GetLLVMFunctionTypeFromSignature(StringHandle libraryfunc)
{
	FunctionSignatureSet::const_iterator iter = ExecContext.LibraryFunctionSignatures.find(libraryfunc);
	if(iter == ExecContext.LibraryFunctionSignatures.end())
		throw FatalException("Invalid library function");

	return GetLLVMFunctionTypeFromSignature(iter->second);
}

llvm::FunctionType* NativeCodeGenerator::GetLLVMFunctionTypeFromSignature(const FunctionSignature& sig)
{
	Type* rettype = GetLLVMType(sig.GetReturnType());

	std::vector<Type*> args;
	for(size_t i = 0; i < sig.GetNumParameters(); ++i)
	{
		const CompileTimeParameter& param = sig.GetParameter(i);
		if(param.HasPayload)
			continue;

		Type* argtype = GetLLVMType(param.Type);
		args.push_back(argtype);
	}

	return FunctionType::get(rettype, args, false);
}

llvm::FunctionType* NativeCodeGenerator::GetLLVMFunctionTypeFromEpochType(Metadata::EpochTypeID type)
{
	return GetLLVMFunctionTypeFromSignature(ExecContext.GetFunctionSignatureByType(type));
}

//
// Add a standard Epoch function implementation to the JIT module
//
void NativeCodeGenerator::AddFunction(size_t beginoffset, size_t endoffset, StringHandle alias)
{
	FunctionJITHelper jithelper(*this);
	jithelper.DoFunction(beginoffset, endoffset, alias);
}

//
// Add a global initialization entity to the JIT module
//
void NativeCodeGenerator::AddGlobalEntity(size_t beginoffset, StringHandle alias)
{
	FunctionJITHelper jithelper(*this);
	jithelper.DoGlobalInit(beginoffset, alias);
}

EPOCHRUNTIME void NativeCodeGenerator::ExternalInvoke(JIT::JITContext& context, StringHandle alias)
{
	Function* func = GetExternalFunction(alias);

	if(ExecContext.SuppressGC.find(alias) == ExecContext.SuppressGC.end())
		Builder.CreateCall(Data->BuiltInFunctions[JITFunc_Runtime_GCBookmark]);

	Function::arg_iterator argiter = context.InnerFunction->arg_begin();
	std::vector<Value*> args;

	Metadata::EpochTypeID rettype = Metadata::EpochType_Void;

	const ScopeDescription& scope = ExecContext.GetScopeDescription(alias);
	for(size_t i = 0; i < scope.GetVariableCount(); ++i)
	{
		if(scope.GetVariableOrigin(i) == VARIABLE_ORIGIN_PARAMETER)
		{
			Metadata::EpochTypeID vartype = scope.GetVariableTypeByIndex(i);

			Value* arg = (Argument*)(argiter);
			if(Metadata::IsReferenceType(vartype) && (Metadata::IsStructureType(vartype) || Metadata::MakeNonReferenceType(vartype) == Metadata::EpochType_Buffer))
				arg = Builder.CreateLoad(arg);

			args.push_back(MarshalArgument(arg, vartype));
			++argiter;
		}
		else if(scope.GetVariableOrigin(i) == VARIABLE_ORIGIN_RETURN)
			rettype = scope.GetVariableTypeByIndex(i);
	}

	CallInst* ret = Builder.CreateCall(func, args);
	ret->setCallingConv(func->getCallingConv());
	if(func->getFunctionType()->getReturnType() != Type::getVoidTy(Data->Context))
		Builder.CreateStore(MarshalReturn(ret, rettype), context.InnerRetVal);

	size_t index = 0;
	for(size_t i = 0; i < scope.GetVariableCount(); ++i)
	{
		if(scope.GetVariableOrigin(i) == VARIABLE_ORIGIN_PARAMETER)
		{
			Metadata::EpochTypeID vartype = scope.GetVariableTypeByIndex(i);
			if(Metadata::IsReferenceType(vartype))
				MarshalReferencePostCall(args[index], context.VariableMap[i], vartype);

			MarshalCleanup(args[index], vartype);

			++index;
		}
	}

	if(ExecContext.SuppressGC.find(alias) == ExecContext.SuppressGC.end())
		Builder.CreateCall(Data->BuiltInFunctions[JITFunc_Runtime_GCUnbookmark]);
}

//
// Optimize LLVM bitcode and generate final native machine code
//
void NativeCodeGenerator::Generate()
{
	for(std::map<Value*, Function*>::const_iterator iter = GeneratedCallbackWrappers.begin(); iter != GeneratedCallbackWrappers.end(); ++iter)
	{
		BasicBlock* block = BasicBlock::Create(Data->Context, "wrap", iter->second);
		Builder.SetInsertPoint(block);

		StringHandle alias = Data->GeneratedFunctionToNameMap.find(iter->second)->second;

		std::vector<Value*> args;
		Function::arg_iterator argiter = iter->second->arg_begin();
		const ScopeDescription& desc = ExecContext.GetScopeDescription(alias);
		size_t paramindex = 0;
		for(size_t i = 0; i < desc.GetParameterCount(); ++i)
		{
			while(desc.GetVariableOrigin(paramindex) != VARIABLE_ORIGIN_PARAMETER)
				++paramindex;

			Value* val = (Argument*)(argiter);
			val = MarshalArgumentReverse(val, desc.GetVariableTypeByIndex(paramindex));

			args.push_back(val);
			++argiter;
		}

		if(iter->second->getFunctionType()->getReturnType() == Type::getVoidTy(Data->Context))
			Builder.CreateRetVoid();
		else
			Builder.CreateRet(MarshalReturnReverse(Builder.CreateLoad(iter->first), desc.GetReturnVariableType()));
	}

	// This dump can come in handy if verification fails or we otherwise
	// need to check up on the bitcode being generated by the JIT engine
	//Data->CurrentModule->dump();

	// Always verify - it is useful for catching JIT bugs
	verifyModule(*Data->CurrentModule, AbortProcessAction);

#ifdef _DEBUG
	//EnableStatistics();
#endif

	std::string ErrStr;

	TargetOptions opts;
	opts.LessPreciseFPMADOption = true;
	opts.UnsafeFPMath = true;
	opts.AllowFPOpFusion = FPOpFusion::Fast;
	opts.DisableTailCalls = false;
	opts.EnableFastISel = false;
	opts.EnableSegmentedStacks = false;
	opts.GuaranteedTailCallOpt = true;

	// Turning off frame pointer elimination can make debugging a LOT smoother...
	opts.NoFramePointerElim = true;
	opts.NoFramePointerElimNonLeaf = true;

	EngineBuilder eb(Data->CurrentModule);
	eb.setEngineKind(EngineKind::JIT);
	eb.setErrorStr(&ErrStr);
	eb.setRelocationModel(Reloc::Default);
	eb.setCodeModel(CodeModel::JITDefault);
	eb.setAllocateGVsWithCode(true);
	eb.setOptLevel(CodeGenOpt::Aggressive);
	eb.setTargetOptions(opts);

	SmallVector<std::string, 2> emptyvec;
	TargetMachine* machine = eb.selectTarget(Triple(), "x86", "corei7-avx", emptyvec);

	ExecutionEngine* ee = eb.create(machine);
	if(!ee)
		return;

	LazyEngine = ee;

	ExecContext.JITExecutionEngine = ee;

	ee->DisableLazyCompilation(true);
	ee->RegisterJITEventListener(this);
	
	FunctionPassManager fpm(Data->CurrentModule);

	fpm.add(new DataLayout(*ee->getDataLayout()));
	fpm.add(createTypeBasedAliasAnalysisPass());
	fpm.add(createBasicAliasAnalysisPass());
	fpm.add(createCFGSimplificationPass());
	fpm.add(createScalarReplAggregatesPass());
	fpm.add(createEarlyCSEPass());
	fpm.add(createLowerExpectIntrinsicPass());

	fpm.doInitialization();

	for(std::map<std::string, Function*>::const_iterator iter = Data->GeneratedFunctions.begin(); iter != Data->GeneratedFunctions.end(); ++iter)
		fpm.run(*iter->second);

	PassManager mpm;
	mpm.add(new DataLayout(*ee->getDataLayout()));
	mpm.add(createTypeBasedAliasAnalysisPass());
	mpm.add(createBasicAliasAnalysisPass());
	mpm.add(createGlobalOptimizerPass());
	mpm.add(createPromoteMemoryToRegisterPass());
	mpm.add(createIPSCCPPass());
	mpm.add(createDeadArgEliminationPass());
	mpm.add(createInstructionCombiningPass());
	mpm.add(createCFGSimplificationPass());
	mpm.add(createPruneEHPass());
	mpm.add(createFunctionAttrsPass());
	mpm.add(createFunctionInliningPass());
	mpm.add(createArgumentPromotionPass());
	mpm.add(createScalarReplAggregatesPass(-1, false));
	mpm.add(createEarlyCSEPass());
	mpm.add(createSimplifyLibCallsPass());
	mpm.add(createJumpThreadingPass());
	mpm.add(createCorrelatedValuePropagationPass());
	mpm.add(createCFGSimplificationPass());
	mpm.add(createInstructionCombiningPass());
	mpm.add(createTailCallEliminationPass());
	mpm.add(createCFGSimplificationPass());
	mpm.add(createReassociatePass());
	mpm.add(createLoopRotatePass());
	mpm.add(createLICMPass());
	mpm.add(createLoopUnswitchPass(false));
	mpm.add(createInstructionCombiningPass());
	mpm.add(createIndVarSimplifyPass());
	mpm.add(createLoopIdiomPass());
	mpm.add(createLoopDeletionPass());
	mpm.add(createLoopUnrollPass());
	mpm.add(createGVNPass());
	mpm.add(createMemCpyOptPass());
	mpm.add(createSCCPPass());
	mpm.add(createInstructionCombiningPass());
	mpm.add(createJumpThreadingPass());
	mpm.add(createCorrelatedValuePropagationPass());
	mpm.add(createDeadStoreEliminationPass());
	mpm.add(createAggressiveDCEPass());
	mpm.add(createCFGSimplificationPass());
	mpm.add(createInstructionCombiningPass());
	mpm.add(createFunctionInliningPass());
	mpm.add(createDeadStoreEliminationPass());

	mpm.add(createSLPVectorizerPass());
	VectorizeConfig vcfg;
	vcfg.ReqChainDepth = 1;
	vcfg.MaxIter = 500;
	mpm.add(createBBVectorizePass(vcfg));

	mpm.run(*Data->CurrentModule);
	
	// This dump is useful for observing the optimized LLVM bitcode
	//Data->CurrentModule->dump();

	Runtime::PopulateWeakLinkages(ExternalFunctions, ee);

	// Perform actual native code generation and link the created
	// pages of machine code back to the runtime for execution
	for(std::map<std::string, Function*>::const_iterator iter = Data->GeneratedFunctions.begin(); iter != Data->GeneratedFunctions.end(); ++iter)
	{
		void* p = ee->getPointerToFunction(iter->second);
		if(iter->second == Data->EntryPoint)
			ExecContext.EntryPointFunc = p;
		else if(iter->second == Data->GlobalInit)
			ExecContext.GlobalInitFunc = p;

		ExecContext.GeneratedJITFunctionCode[p] = std::make_pair(iter->second, Data->GeneratedFunctionToNameMap[iter->second]);
		ExecContext.GeneratedFunctionLLVMToMachineCodeMap[iter->second] = p;
	}

	for(Module::const_global_iterator giter = Data->CurrentModule->global_begin(); giter != Data->CurrentModule->global_end(); ++giter)
	{
		const GlobalVariable* global = dyn_cast<const GlobalVariable>((const GlobalValue*)giter);
		if(global)
			EpochGC::RegisterGlobalVariable(ee->getPointerToGlobal(global), Data->GlobalVariableTypes[global]);
	}

	// This is a no-op unless we enabled stats above
	// The numbers are very handy for A/B testing optimization passes
	PrintStatistics();
}

Function* NativeCodeGenerator::GetExternalFunction(StringHandle alias)
{
	Function* func = ExternalFunctions[alias];
	if(!func)
	{
		const Runtime::DLLInvocationInfo& invokeinfo = Runtime::GetMarshaledExternalFunction(alias);
		std::string name = narrow(invokeinfo.DLLName) + "_" + narrow(invokeinfo.FunctionName);

		FunctionType* ftype = GetExternalFunctionType(alias);
		func = Function::Create(ftype, GlobalValue::ExternalWeakLinkage, name, Data->CurrentModule);

		if(invokeinfo.CallingConvention == L"stdcall")
			func->setCallingConv(CallingConv::X86_StdCall);

		ExternalFunctions[alias] = func;
	}

	return func;
}



//
// Construct and initialize a function JIT helper object
//
FunctionJITHelper::FunctionJITHelper(NativeCodeGenerator& generator)
	: Generator(generator),
	  Builder(generator.Builder),
	  Context(generator.Data->Context),
	  Bytecode(generator.Bytecode),
	  CompilingTypeMatchedFunction(false),
	  TypeMatchedParams(NULL),
	  WrapperFunction(NULL)
{
	InstructionJITHelpers[Bytecode::Instructions::BeginEntity] = &FunctionJITHelper::BeginEntity;
	InstructionJITHelpers[Bytecode::Instructions::EndEntity] = &FunctionJITHelper::EndEntity;

	InstructionJITHelpers[Bytecode::Instructions::BeginChain] = &FunctionJITHelper::BeginChain;
	InstructionJITHelpers[Bytecode::Instructions::EndChain] = &FunctionJITHelper::EndChain;

	InstructionJITHelpers[Bytecode::Instructions::Read] = &FunctionJITHelper::Read;
	InstructionJITHelpers[Bytecode::Instructions::ReadStack] = &FunctionJITHelper::ReadStackLocal;
	InstructionJITHelpers[Bytecode::Instructions::ReadParam] = &FunctionJITHelper::ReadParameter;

	InstructionJITHelpers[Bytecode::Instructions::BindRef] = &FunctionJITHelper::BindReference;
	InstructionJITHelpers[Bytecode::Instructions::ReadRef] = &FunctionJITHelper::ReadRef;
	InstructionJITHelpers[Bytecode::Instructions::ReadRefAnnotated] = &FunctionJITHelper::ReadRefAnnotated;

	InstructionJITHelpers[Bytecode::Instructions::Assign] = &FunctionJITHelper::Assign;
	InstructionJITHelpers[Bytecode::Instructions::AssignSumType] = &FunctionJITHelper::AssignSumType;
	InstructionJITHelpers[Bytecode::Instructions::ConstructSumType] = &FunctionJITHelper::ConstructSumType;

	InstructionJITHelpers[Bytecode::Instructions::SetRetVal] = &FunctionJITHelper::SetRetValue;
	InstructionJITHelpers[Bytecode::Instructions::Return] = &FunctionJITHelper::Return;
	InstructionJITHelpers[Bytecode::Instructions::Halt] = &FunctionJITHelper::Halt;

	InstructionJITHelpers[Bytecode::Instructions::Push] = &FunctionJITHelper::Push;
	InstructionJITHelpers[Bytecode::Instructions::Pop] = &FunctionJITHelper::Pop;

	InstructionJITHelpers[Bytecode::Instructions::AllocStructure] = &FunctionJITHelper::AllocStructure;
	InstructionJITHelpers[Bytecode::Instructions::CopyToStructure] = &FunctionJITHelper::CopyToStructure;
	InstructionJITHelpers[Bytecode::Instructions::CopyStructure] = &FunctionJITHelper::CopyStructure;
	InstructionJITHelpers[Bytecode::Instructions::BindMemberRef] = &FunctionJITHelper::BindMemberRef;

	InstructionJITHelpers[Bytecode::Instructions::Invoke] = &FunctionJITHelper::Invoke;
	InstructionJITHelpers[Bytecode::Instructions::InvokeOffset] = &FunctionJITHelper::InvokeOffset;
	InstructionJITHelpers[Bytecode::Instructions::InvokeNative] = &FunctionJITHelper::InvokeNative;
	InstructionJITHelpers[Bytecode::Instructions::InvokeIndirect] = &FunctionJITHelper::InvokeIndirect;

	InstructionJITHelpers[Bytecode::Instructions::TypeMatch] = &FunctionJITHelper::TypeMatch;
	InstructionJITHelpers[Bytecode::Instructions::PatternMatch] = &FunctionJITHelper::PatternMatch;

	InstructionJITHelpers[Bytecode::Instructions::CopyBuffer] = &FunctionJITHelper::CopyBuffer;
}


AllocaInst* FunctionJITHelper::CreateAllocaInternal(Type* type, bool forceintofirstblock)
{
	AllocaInst* ret;
	if(CompilingTypeMatchedFunction || forceintofirstblock)
		ret = new AllocaInst(type, "", &Builder.GetInsertBlock()->getParent()->getEntryBlock().back());
	else
		ret = Builder.CreateAlloca(type);

	return ret;
}

//
// JIT a function
//
void FunctionJITHelper::DoFunction(size_t beginoffset, size_t endoffset, StringHandle alias)
{
	// Set up context for external libraries that we need to interact with
	// TODO - clean up JITContext a bit
	LibJITContext.Builder = &Builder;
	LibJITContext.Context = &Context;
	LibJITContext.MyModule = Generator.Data->CurrentModule;
	LibJITContext.InnerFunction = NULL;
	LibJITContext.VarArgList = NULL;
	LibJITContext.Generator = &Generator;
	LibJITContext.FunctionAlias = alias;

	LibJITContext.BuiltInFunctions = &Generator.Data->BuiltInFunctions;

	while(!LibJITContext.ValuesOnStack.empty())
		LibJITContext.ValuesOnStack.pop();

	// Merge in globals
	for(std::map<StringHandle, size_t>::const_iterator iter = Generator.Data->GlobalVariableNameToIndexMap.begin(); iter != Generator.Data->GlobalVariableNameToIndexMap.end(); ++iter)
	{
		size_t index = iter->second + 0xf0000000;
		LibJITContext.VariableMap[index] = Generator.Data->GlobalVariableMap[iter->second];
		LibJITContext.NameToIndexMap[iter->first] = index;
	}

	// Initialize tracking for JIT operations
	LibJITContext.CurrentScope = NULL;

	NumParameters = 0;
	NumParameterBytes = 0;
	NumReturns = 0;

	AllocCount = 0;

	HackStructType = 0;

	ChainDepth = 0;

	BeginOffset = beginoffset;
	EndOffset = endoffset;


	// Initialize block pointers that are lazily populated
	LibJITContext.InnerExitBlock = NULL;
	NativeMatchBlock = NULL;
	PatternMatchBlock = NULL;

	// Initialize values used during JIT procedures
	LibJITContext.InnerRetVal = NULL;

	// Now process each instruction in the Epoch bytecode and produce the LLVM bitcode output
	for(size_t offset = beginoffset; offset <= endoffset; )
	{
		Bytecode::Instruction instruction = Bytecode[offset++];

		if(InstructionJITHelpers.find(instruction) == InstructionJITHelpers.end())
			throw FatalException("Invalid instruction for native code generation");

		InstructionJITHelper helper = InstructionJITHelpers[instruction];
		(this->*helper)(offset);
	}
	
	if(LibJITContext.InnerFunction)
	{
		Builder.SetInsertPoint(LibJITContext.InnerExitBlock);

		if(LibJITContext.VarArgList)
			Builder.CreateCall(Generator.Data->BuiltInFunctions[JITFunc_Intrinsic_VAEnd], Builder.CreatePointerCast(LibJITContext.VarArgList, Type::getInt8PtrTy(Context)));

		if(Builder.GetInsertBlock()->getParent()->hasGC())// && AllocCount > 0)
			Builder.CreateCall(Generator.Data->BuiltInFunctions[JITFunc_Runtime_TriggerGC]);

#ifdef PROFILING_HOOKS
		Builder.CreateCall(Generator.Data->BuiltInFunctions[JITFunc_Profile_Exit], ConstantInt::get(Type::getInt32Ty(Generator.Data->Context), alias));
#endif

		// TODO - this is kind of hacky
		if(Generator.ExecContext.GetPooledString(alias) == L"entrypoint")
		{
			Generator.Data->EntryPoint = LibJITContext.InnerFunction;

#ifdef PROFILING_HOOKS
			Builder.CreateCall(Generator.Data->BuiltInFunctions[JITFunc_Profile_Dump]);
#endif
		}

		if(LibJITContext.InnerRetVal)
		{
			Value* ret = Builder.CreateLoad(LibJITContext.InnerRetVal);
			Builder.CreateRet(ret);
		}
		else
			Builder.CreateRetVoid();

		Generator.Data->GeneratedNameToFunctionMap[alias] = LibJITContext.InnerFunction;
	}

	if(NativeMatchBlock)
	{
		Builder.SetInsertPoint(NativeMatchBlock);
		Generator.AddNativeTypeMatcher(beginoffset, endoffset);
	}
	else if(PatternMatchBlock)
	{
		Builder.SetInsertPoint(PatternMatchBlock);
		Generator.AddNativePatternMatcher(beginoffset, endoffset);
	}
}


void FunctionJITHelper::DoTypeMatchedFunction(size_t beginoffset, size_t endoffset, StringHandle alias, const std::vector<Value*>& params, Function* wrap)
{
	CompilingTypeMatchedFunction = true;
	TypeMatchedParams = &params;
	WrapperFunction = wrap;

	DoFunction(beginoffset, endoffset, alias);

	CompilingTypeMatchedFunction = false;
	TypeMatchedParams = NULL;
	WrapperFunction = NULL;
}


//
// Process a global variable initialization block
//
void FunctionJITHelper::DoGlobalInit(size_t beginoffset, StringHandle alias)
{
	LibJITContext.Builder = &Builder;
	LibJITContext.Context = &Context;
	LibJITContext.MyModule = Generator.Data->CurrentModule;
	LibJITContext.InnerFunction = NULL;
	LibJITContext.VarArgList = NULL;
	LibJITContext.Generator = &Generator;
	LibJITContext.FunctionAlias = 0;

	LibJITContext.BuiltInFunctions = &Generator.Data->BuiltInFunctions;

	// Initialize tracking for JIT operations
	LibJITContext.CurrentScope = &Generator.ExecContext.GetScopeDescription(alias);

	NumParameters = 0;
	NumParameterBytes = 0;
	NumReturns = 0;

	HackStructType = 0;

	// Initialize block pointers that are lazily populated
	LibJITContext.InnerExitBlock = NULL;
	NativeMatchBlock = NULL;
	PatternMatchBlock = NULL;

	// Initialize values used during JIT procedures
	LibJITContext.InnerRetVal = NULL;

	// Now process each instruction in the Epoch bytecode and produce the LLVM bitcode output
	size_t offset = beginoffset;
	for(;;)
	{
		Bytecode::Instruction instruction = Bytecode[offset++];
		if(instruction == Bytecode::Instructions::Halt)
			break;

		if(InstructionJITHelpers.find(instruction) == InstructionJITHelpers.end())
			throw FatalException("Invalid instruction for native code generation");

		InstructionJITHelper helper = InstructionJITHelpers[instruction];
		(this->*helper)(offset);
	}
	
	if(LibJITContext.InnerFunction)
	{
		Builder.CreateBr(LibJITContext.InnerExitBlock);
		Builder.SetInsertPoint(LibJITContext.InnerExitBlock);
		Builder.CreateRetVoid();

		Generator.Data->GlobalInit = LibJITContext.InnerFunction;
	}
}

//
// Convert the Epoch instruction for entering an entity into LLVM bitcode
//
// The significance of Epoch entities vis-a-vis LLVM basic blocks is interesting.
// Both concepts are similar in many respects, with the principal distinction lying
// in the fact that Epoch entities and entity chains can attach arbitrary meta-control
// logic to the execution flow of the entity bodies themselves. In order to allow the
// Epoch standard library entities (and potentially third-party entities) to behave
// correctly in JITted native code, we offer a helper interface which shells out to
// the library implementation to emit meta-control code that mimics the Epoch entity
// behavior. This structure allows us to maintain the flexibility and expressive power
// potential of the entity system without compromising the JIT engine's functionality.
//
void FunctionJITHelper::BeginEntity(size_t& offset)
{
	Bytecode::EntityTag entitytype = Fetch<Integer32>(Bytecode, offset);
	LibJITContext.EntityTypes.push(entitytype);

	StringHandle entityname = Fetch<StringHandle>(Bytecode, offset);

	if(entitytype == Bytecode::EntityTags::Function)
	{
		Type* rettype = Type::getVoidTy(Context);
		Type* type = NULL;

		std::set<size_t> locals;
		std::set<size_t> parameters;

		std::map<size_t, size_t> paramindices;

		size_t localoffsetbytes = 0;

		size_t retindex = static_cast<size_t>(-1);

		const ScopeDescription& scope = Generator.ExecContext.GetScopeDescription(entityname);
		LibJITContext.CurrentScope = &scope;

		for(size_t i = scope.GetVariableCount(); i-- > 0; )
		{
			Metadata::EpochTypeID vartype = scope.GetVariableTypeByIndex(i);

			if(vartype == Metadata::EpochType_Nothing)
			{
				if(scope.GetVariableOrigin(i) == VARIABLE_ORIGIN_PARAMETER)
				{
					parameters.insert(i);
					paramindices[NumParameters] = i;
					++NumParameters;
				}
				continue;
			}

			type = Generator.GetLLVMType(vartype);

			switch(scope.GetVariableOrigin(i))
			{
			case VARIABLE_ORIGIN_RETURN:
				++NumReturns;
				retindex = i;
				rettype = type;
				// Deliberate fallthrough

			case VARIABLE_ORIGIN_LOCAL:
				locals.insert(i);
				break;

			case VARIABLE_ORIGIN_PARAMETER:
				{
					ParameterOffsetToIndexMap[NumParameterBytes] = i;
					parameters.insert(i);

					paramindices[NumParameters] = i;

					if(Metadata::IsReferenceType(vartype))
						NumParameterBytes += sizeof(void*) + sizeof(Metadata::EpochTypeID);
					else
					{
						if(Metadata::GetTypeFamily(vartype) == Metadata::EpochTypeFamily_SumType)
							NumParameterBytes += Generator.ExecContext.VariantDefinitions.find(vartype)->second.GetMaxSize();
						else
							NumParameterBytes += Metadata::GetStorageSize(vartype);
					}

					++NumParameters;
				}
				break;
			}

			LibJITContext.NameToIndexMap[scope.GetVariableNameHandle(i)] = i;
		}

		if(!CompilingTypeMatchedFunction)
		{
			LibJITContext.InnerFunction = Generator.GetGeneratedFunction(entityname, BeginOffset);
		}
		else
		{
			LibJITContext.InnerFunction = WrapperFunction;
		}

		LibJITContext.InnerEntryBlock = BasicBlock::Create(Context, "innerentry", LibJITContext.InnerFunction);
		Builder.SetInsertPoint(LibJITContext.InnerEntryBlock);

#ifdef PROFILING_HOOKS
		Builder.CreateCall(Generator.Data->BuiltInFunctions[JITFunc_Profile_Enter], ConstantInt::get(Type::getInt32Ty(Generator.Data->Context), entityname));
#endif

		LibJITContext.InnerExitBlock = BasicBlock::Create(Context, "innerexit", LibJITContext.InnerFunction);

		if(NumReturns)
			LibJITContext.InnerRetVal = CreateAllocaInternal(rettype);

		if(!CompilingTypeMatchedFunction)
		{
			size_t idx = NumParameters;
			Function::ArgumentListType& args = LibJITContext.InnerFunction->getArgumentList();
			for(Function::ArgumentListType::iterator argiter = args.begin(); argiter != args.end(); ++argiter)
			{
				--idx;
				Metadata::EpochTypeID epochtype = LibJITContext.CurrentScope->GetVariableTypeByIndex(paramindices[idx]);

				if(Metadata::IsReferenceType(epochtype))
				{
					LibJITContext.VariableMap[paramindices[idx]] = ((Argument*)argiter);

					if(LibJITContext.InnerFunction->hasGC())
					{
						if((Metadata::GetTypeFamily(epochtype) == Metadata::EpochTypeFamily_GC) || (Metadata::GetTypeFamily(epochtype) == Metadata::EpochTypeFamily_SumType) || Metadata::IsStructureType(epochtype))
						{
							Value* deref = Builder.CreateLoad((Argument*)(argiter));

							Value* slot = CreateAllocaInternal(deref->getType());
							Builder.CreateStore(deref, slot);

							Value* signature = ConstantInt::get(Type::getInt32Ty(Context), epochtype);
							Value* constant = Builder.CreateIntToPtr(signature, Type::getInt8PtrTy(Context));
							Value* castptr = Builder.CreatePointerCast(slot, Type::getInt8PtrTy(Context)->getPointerTo());
							Builder.CreateCall2(Generator.Data->BuiltInFunctions[JITFunc_Intrinsic_GCRoot], castptr, constant);
						}
					}
				}
				else
				{
					Value* slot = CreateAllocaInternal(((Argument*)argiter)->getType());
					Builder.CreateStore(((Argument*)argiter), slot);

					if(LibJITContext.InnerFunction->hasGC())
					{
						if((Metadata::GetTypeFamily(epochtype) == Metadata::EpochTypeFamily_GC) || (Metadata::GetTypeFamily(epochtype) == Metadata::EpochTypeFamily_SumType) || Metadata::IsStructureType(epochtype))
						{
							Value* signature = ConstantInt::get(Type::getInt32Ty(Context), epochtype);
							Value* constant = Builder.CreateIntToPtr(signature, Type::getInt8PtrTy(Context));
							Value* castptr = Builder.CreatePointerCast(slot, Type::getInt8PtrTy(Context)->getPointerTo());
							Builder.CreateCall2(Generator.Data->BuiltInFunctions[JITFunc_Intrinsic_GCRoot], castptr, constant);
						}
					}

					LibJITContext.VariableMap[paramindices[idx]] = slot;
				}
			}
		}
		else
		{
			size_t idx = NumParameters;
			for(std::vector<Value*>::const_iterator argiter = TypeMatchedParams->begin(); argiter != TypeMatchedParams->end(); ++argiter)
			{
				--idx;
				Metadata::EpochTypeID epochtype = LibJITContext.CurrentScope->GetVariableTypeByIndex(paramindices[idx]);

				if(Metadata::IsReferenceType(epochtype))
				{
					LibJITContext.VariableMap[paramindices[idx]] = (*argiter);

					if(LibJITContext.InnerFunction->hasGC())
					{
						if((Metadata::GetTypeFamily(epochtype) == Metadata::EpochTypeFamily_GC) || (Metadata::GetTypeFamily(epochtype) == Metadata::EpochTypeFamily_SumType) || Metadata::IsStructureType(epochtype))
						{
							Value* deref = Builder.CreateLoad(*argiter);

							Value* slot = CreateAllocaInternal(deref->getType());
							Builder.CreateStore(deref, slot);

							Value* signature = ConstantInt::get(Type::getInt32Ty(Context), epochtype);
							Value* constant = Builder.CreateIntToPtr(signature, Type::getInt8PtrTy(Context));
							Value* castptr = Builder.CreatePointerCast(slot, Type::getInt8PtrTy(Context)->getPointerTo());
							Builder.CreateCall2(Generator.Data->BuiltInFunctions[JITFunc_Intrinsic_GCRoot], castptr, constant);
						}
					}
				}
				else
				{
					Value* slot = CreateAllocaInternal((*argiter)->getType());
					Builder.CreateStore((*argiter), slot);

					if(LibJITContext.InnerFunction->hasGC())
					{
						if((Metadata::GetTypeFamily(epochtype) == Metadata::EpochTypeFamily_GC) || (Metadata::GetTypeFamily(epochtype) == Metadata::EpochTypeFamily_SumType) || Metadata::IsStructureType(epochtype))
						{
							Value* signature = ConstantInt::get(Type::getInt32Ty(Context), epochtype);
							Value* constant = Builder.CreateIntToPtr(signature, Type::getInt8PtrTy(Context));
							Value* castptr = Builder.CreatePointerCast(slot, Type::getInt8PtrTy(Context)->getPointerTo());
							Builder.CreateCall2(Generator.Data->BuiltInFunctions[JITFunc_Intrinsic_GCRoot], castptr, constant);
						}
					}

					LibJITContext.VariableMap[paramindices[idx]] = slot;
				}
			}
		}

		if(NumReturns)
			LibJITContext.VariableMap[retindex] = LibJITContext.InnerRetVal;

		localoffsetbytes = 0;
		for(std::set<size_t>::const_iterator localiter = locals.begin(); localiter != locals.end(); ++localiter)
		{
			Metadata::EpochTypeID localtype = scope.GetVariableTypeByIndex(*localiter);
			Type* type = Generator.GetLLVMType(localtype);

			if(*localiter != retindex)
				LibJITContext.VariableMap[*localiter] = CreateAllocaInternal(type);

			LocalOffsetToIndexMap[localoffsetbytes] = *localiter;

			// TODO - only flag GC roots of sum type when the sum type might contain a GC-able type
			if(Builder.GetInsertBlock()->getParent()->hasGC())
			{
				if((Metadata::GetTypeFamily(localtype) == Metadata::EpochTypeFamily_GC) || (Metadata::GetTypeFamily(localtype) == Metadata::EpochTypeFamily_SumType) || Metadata::IsStructureType(localtype))
				{
					Value* signature = ConstantInt::get(Type::getInt32Ty(Context), localtype);
					Value* constant = Builder.CreateIntToPtr(signature, Type::getInt8PtrTy(Context));

					Value* castptr = Builder.CreatePointerCast(LibJITContext.VariableMap[*localiter], Type::getInt8PtrTy(Context)->getPointerTo());
					Builder.CreateCall2(Generator.Data->BuiltInFunctions[JITFunc_Intrinsic_GCRoot], castptr, constant);
				}
			}

			if(Metadata::GetTypeFamily(localtype) == Metadata::EpochTypeFamily_SumType)
				localoffsetbytes += Generator.ExecContext.VariantDefinitions.find(localtype)->second.GetMaxSize();
			else
				localoffsetbytes += Metadata::GetStorageSize(localtype);
		}
	}
	else if(entitytype == Bytecode::EntityTags::TypeResolver)
	{
		Function* nativetypematcher = Generator.GetGeneratedTypeMatcher(entityname, BeginOffset);
		NativeMatchBlock = BasicBlock::Create(Context, "nativematchentry", nativetypematcher);
	}
	else if(entitytype == Bytecode::EntityTags::PatternMatchingResolver)
	{
		Function* nativematcher = Generator.GetGeneratedPatternMatcher(entityname, BeginOffset);
		PatternMatchBlock = BasicBlock::Create(Context, "nativepatternmatchentry", nativematcher);
	}
	else if(entitytype == Bytecode::EntityTags::Globals)
	{
		const ScopeDescription& scope = Generator.ExecContext.GetScopeDescription(entityname);

		// Sanity check
		for(size_t i = scope.GetVariableCount(); i-- > 0; )
		{
			switch(scope.GetVariableOrigin(i))
			{
			case VARIABLE_ORIGIN_PARAMETER:
			case VARIABLE_ORIGIN_RETURN:
				throw FatalException("Global scope cannot contain parameters or return values");
			}
		}

		LibJITContext.InnerFunction = Generator.GetGeneratedGlobalInit(entityname);

		BasicBlock* innerentryblock = BasicBlock::Create(Context, "innerentry", LibJITContext.InnerFunction);
		Builder.SetInsertPoint(innerentryblock);

		LibJITContext.InnerExitBlock = BasicBlock::Create(Context, "innerexit", LibJITContext.InnerFunction);

		size_t localoffsetbytes = 0;
		for(size_t i = 0; i < scope.GetVariableCount(); ++i)
		{
			Metadata::EpochTypeID localtype = scope.GetVariableTypeByIndex(i);
			Type* type = Generator.GetLLVMType(localtype);

			Constant* init = NULL;

			switch(localtype)
			{
			case Metadata::EpochType_Integer:
				init = ConstantInt::get(Type::getInt32Ty(Context), 0); 
				break;

			case Metadata::EpochType_String:
				init = ConstantInt::get(Type::getInt32Ty(Context), 0); 
				break;

			case Metadata::EpochType_Buffer:
				init = ConstantInt::get(Type::getInt32Ty(Context), 0); 
				break;

			case Metadata::EpochType_Boolean:
				init = ConstantInt::get(Type::getInt1Ty(Context), 0);
				break;

			default:
				{
					if(Metadata::IsStructureType(localtype))
					{
						init = ConstantPointerNull::get(Type::getInt8PtrTy(Context));
						break;
					}
				}
				throw NotImplementedException("Unsupported global variable type");
			}

			Generator.Data->GlobalVariableMap[i] = new GlobalVariable(*Generator.Data->CurrentModule, type, false, GlobalValue::InternalLinkage, init, narrow(Generator.ExecContext.GetPooledString(scope.GetVariableNameHandle(i))));
			Generator.Data->GlobalVariableTypes[Generator.Data->GlobalVariableMap[i]] = localtype;
			Generator.Data->GlobalVariableOffsetToIndexMap[localoffsetbytes] = i;
			Generator.Data->GlobalVariableNameToIndexMap[scope.GetVariableNameHandle(i)] = i;

			if(Metadata::GetTypeFamily(localtype) == Metadata::EpochTypeFamily_SumType)
				localoffsetbytes += Generator.ExecContext.VariantDefinitions.find(localtype)->second.GetMaxSize();
			else
				localoffsetbytes += Metadata::GetStorageSize(localtype);
		}

		for(std::map<StringHandle, size_t>::const_iterator iter = Generator.Data->GlobalVariableNameToIndexMap.begin(); iter != Generator.Data->GlobalVariableNameToIndexMap.end(); ++iter)
		{
			size_t index = iter->second;
			LibJITContext.VariableMap[index] = Generator.Data->GlobalVariableMap[iter->second];
			LibJITContext.VariableMap[index + 0xf0000000] = Generator.Data->GlobalVariableMap[iter->second];
			LibJITContext.NameToIndexMap[iter->first] = index;
		}
	}
	else
	{
		std::map<StringHandle, JIT::JITHelper>::const_iterator helperiter = Generator.ExecContext.JITHelpers.EntityHelpers.find(entitytype);
		if(helperiter == Generator.ExecContext.JITHelpers.EntityHelpers.end())
			throw FatalException("Cannot JIT this type of entity");
					
		helperiter->second(LibJITContext, true);
	}
}

//
// Convert the Epoch instruction for exiting an entity into LLVM bitcode
//
void FunctionJITHelper::EndEntity(size_t&)
{
	Bytecode::EntityTag tag = LibJITContext.EntityTypes.top();
	LibJITContext.EntityTypes.pop();

	if(tag != Bytecode::EntityTags::Function && tag != Bytecode::EntityTags::TypeResolver && tag != Bytecode::EntityTags::PatternMatchingResolver)
		Generator.ExecContext.JITHelpers.EntityHelpers.find(tag)->second(LibJITContext, false);
}

//
// Convert the Epoch instruction for entering an entity chain into LLVM bitcode
//
void FunctionJITHelper::BeginChain(size_t&)
{
	++ChainDepth;
	LibJITContext.EntityChecks.push(BasicBlock::Create(Context, "entitycheck", LibJITContext.InnerFunction));
	LibJITContext.EntityChains.push(BasicBlock::Create(Context, "entitychain", LibJITContext.InnerFunction));
	LibJITContext.EntityChainExits.push(BasicBlock::Create(Context, "entitychainexit", LibJITContext.InnerFunction));
	Builder.CreateBr(LibJITContext.EntityChecks.top());
	Builder.SetInsertPoint(LibJITContext.EntityChecks.top());
}

//
// Convert the Epoch instruction for exiting an entity chain into LLVM bitcode
//
void FunctionJITHelper::EndChain(size_t&)
{
	--ChainDepth;

	if(LibJITContext.EntityChains.top()->empty())
	{
		Builder.SetInsertPoint(LibJITContext.EntityChains.top());
		Builder.CreateBr(LibJITContext.EntityChainExits.top());
	}

	for(std::vector<std::pair<llvm::BasicBlock*, llvm::BasicBlock*> >::const_reverse_iterator riter = LibJITContext.CheckOrphanBlocks.rbegin(); riter != LibJITContext.CheckOrphanBlocks.rend(); ++riter)
	{
		if(riter->second != LibJITContext.EntityChainExits.top())
			break;

		if(riter->first->empty())
		{
			Builder.SetInsertPoint(riter->first);
			Builder.CreateBr(riter->second);
		}
	}

	LibJITContext.EntityChecks.pop();
	LibJITContext.EntityChains.pop();
	Builder.SetInsertPoint(LibJITContext.EntityChainExits.top());
	LibJITContext.EntityChainExits.pop();
}

//
// Generate LLVM bitcode to read a named variable onto the stack
//
// Note that we do not literally push things onto the machine stack as a
// result of processing this instruction; instead, the JIT engine maintains
// a "virtual stack" which tracks what values would be on the stack if we
// pushed/popped them all individually as the IR requests.
//
// The virtual stack can be very handy in eliminating redundant loads and
// stores, because we don't need to stack-shuffle every single value we want
// to operate on the way the Epoch "machine" does.
//
void FunctionJITHelper::Read(size_t& offset)
{
	StringHandle varname = Fetch<StringHandle>(Bytecode, offset);
	if(LibJITContext.NameToIndexMap.find(varname) == LibJITContext.NameToIndexMap.end())
		throw FatalException("Name not mapped to index");

	size_t index = LibJITContext.NameToIndexMap[varname];
	size_t scopeindex = index;

	const ScopeDescription* scope = LibJITContext.CurrentScope;
	if(index >= 0xf0000000)
	{
		scopeindex -= 0xf0000000;
		scope = scope->ParentScope;
	}

	bool isref = Metadata::IsReferenceType(scope->GetVariableTypeByIndex(scopeindex));
	Metadata::EpochTypeID vartype = Metadata::MakeNonReferenceType(scope->GetVariableTypeByIndex(scopeindex));
	if(Builder.GetInsertBlock()->getParent()->isVarArg() && (scope->GetVariableOrigin(scopeindex) != VARIABLE_ORIGIN_RETURN))
	{
		if(Metadata::GetTypeFamily(vartype) == Metadata::EpochTypeFamily_SumType)
		{
			Value* payload = Builder.CreateVAArg(LibJITContext.VarArgList, Generator.GetLLVMSumType(vartype, true)->getContainedType(1));
			Value* typesignature = Builder.CreateVAArg(LibJITContext.VarArgList, Generator.Data->TypeIDType);

			Value* sumtypeholder = CreateAllocaInternal(Generator.GetLLVMSumType(vartype, true));

			std::vector<Value*> gepindices;
			gepindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 0));
			gepindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 0));
			Value* typeholder = Builder.CreateGEP(sumtypeholder, gepindices);

			std::vector<Value*> payloadgepindices;
			payloadgepindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 0));
			payloadgepindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 1));
			Value* payloadholder = Builder.CreateGEP(sumtypeholder, payloadgepindices);
			
			Builder.CreateStore(typesignature, typeholder);
			Builder.CreateStore(payload, payloadholder);

			Value* ptr = Builder.CreateLoad(sumtypeholder);
			LibJITContext.ValuesOnStack.push(ptr);
			LibJITContext.VariableMap[index] = ptr;
		}
		else
		{
			Type* type = Generator.GetLLVMType(vartype);
			Value* ptr = Builder.CreateVAArg(LibJITContext.VarArgList, type);

			if(isref)
				ptr = Builder.CreateLoad(ptr);

			LibJITContext.ValuesOnStack.push(ptr);
			LibJITContext.VariableMap[index] = ptr;
		}
	}
	else
	{
		if(Metadata::GetTypeFamily(vartype) == Metadata::EpochTypeFamily_SumType)
		{
			Value* v = LibJITContext.VariableMap[index];

			std::vector<Value*> gepindices;
			gepindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 0));
			gepindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 0));
			Value* typeholder = Builder.CreateLoad(Builder.CreateGEP(v, gepindices));

			std::vector<Value*> payloadgepindices;
			payloadgepindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 0));
			payloadgepindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 1));
			Value* payload = Builder.CreateLoad(Builder.CreateGEP(v, payloadgepindices));

			LibJITContext.ValuesOnStack.push(payload);
			LibJITContext.ValuesOnStack.push(typeholder);
		}
		else
		{
			Value* v = Builder.CreateLoad(LibJITContext.VariableMap[index]);
			LibJITContext.ValuesOnStack.push(v);
		}
	}
}

//
// Handle the EpochRuntime special instruction for reading local variables
// onto the Epoch execution stack. Similar to the Read instruction, this
// is handled by using the virtual value stack instead of generating an
// actual stack shuffle on the machine stack.
//
void FunctionJITHelper::ReadStackLocal(size_t& offset)
{
	size_t frames = Fetch<size_t>(Bytecode, offset);
	size_t stackoffset = Fetch<size_t>(Bytecode, offset);
	Fetch<size_t>(Bytecode, offset);		// stack size is irrelevant

	if((frames != 0) && (frames != (unsigned)(-1)))		// TODO - this is an ugly hardcoded hack
		throw NotImplementedException("Scope is not flat!");

	Metadata::EpochTypeID type = Metadata::EpochType_Error;
	Value* v = NULL;
	
	if(frames == 0)
	{
		if(LocalOffsetToIndexMap.find(stackoffset) == LocalOffsetToIndexMap.end())
			throw FatalException("Invalid stack offset");

		size_t index = LocalOffsetToIndexMap[stackoffset];
		type = LibJITContext.CurrentScope->GetVariableTypeByIndex(index);
		v = LibJITContext.VariableMap[index];
	}
	else
	{
		const ScopeDescription* scope = LibJITContext.CurrentScope;
		while(scope->ParentScope)
			scope = scope->ParentScope;

		if(Generator.Data->GlobalVariableOffsetToIndexMap.find(stackoffset) == Generator.Data->GlobalVariableOffsetToIndexMap.end())
			throw FatalException("Invalid global offset");

		size_t index = Generator.Data->GlobalVariableOffsetToIndexMap[stackoffset];
		type = scope->GetVariableTypeByIndex(index);
		v = Generator.Data->GlobalVariableMap[index]; 
	}

	if(Metadata::GetTypeFamily(type) == Metadata::EpochTypeFamily_SumType)
	{
		std::vector<Value*> gepindices;
		gepindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 0));
		gepindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 0));
		Value* typeholder = Builder.CreateLoad(Builder.CreateGEP(v, gepindices));

		std::vector<Value*> payloadgepindices;
		payloadgepindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 0));
		payloadgepindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 1));
		Value* payload = Builder.CreateLoad(Builder.CreateGEP(v, payloadgepindices));

		LibJITContext.ValuesOnStack.push(payload);
		LibJITContext.ValuesOnStack.push(typeholder);
	}
	else
	{
		Value* val = Builder.CreateLoad(v);
		LibJITContext.ValuesOnStack.push(val);
	}
}

//
// Handle another Read instruction variant targeted at function
// parameters. The main quirk here is that we automatically dereference
// values which were passed to the function by ref.
//
void FunctionJITHelper::ReadParameter(size_t& offset)
{
	size_t frames = Fetch<size_t>(Bytecode, offset);
	size_t stackoffset = Fetch<size_t>(Bytecode, offset);
	Fetch<size_t>(Bytecode, offset);		// stack size is irrelevant

	if(frames != 0)
		throw NotImplementedException("Scope is not flat!");
				
	size_t idx = ParameterOffsetToIndexMap[stackoffset];
	if(Metadata::GetTypeFamily(LibJITContext.CurrentScope->GetVariableTypeByIndex(idx)) == Metadata::EpochTypeFamily_SumType)
	{
		Value* v = LibJITContext.VariableMap[idx];

		std::vector<Value*> gepindices;
		gepindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 0));
		gepindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 0));
		Value* typeholder = Builder.CreateLoad(Builder.CreateGEP(v, gepindices));

		std::vector<Value*> payloadgepindices;
		payloadgepindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 0));
		payloadgepindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 1));
		LoadInst* payload = Builder.CreateLoad(Builder.CreateGEP(v, payloadgepindices));
		payload->setAlignment(1);

		LibJITContext.ValuesOnStack.push(payload);
		LibJITContext.ValuesOnStack.push(typeholder);
	}
	else
	{
		LoadInst* val = Builder.CreateLoad(LibJITContext.VariableMap[idx]);
		val->setAlignment(1);
		LibJITContext.ValuesOnStack.push(val);
	}
}

//
// In the Epoch IR, we bind references to things by placing a pointer
// and a type annotation onto the Epoch stack. In the JIT engine, we
// convert this to simply storing a raw pointer (discarding type safety
// since in theory the compiler already ensures that for us) and track
// the type annotation in a virtual stack. This again eliminates a lot
// of load/store activity on the machine stack.
//
void FunctionJITHelper::BindReference(size_t& offset)
{
	size_t frames = Fetch<size_t>(Bytecode, offset);
	size_t index = Fetch<size_t>(Bytecode, offset);

	if((frames != 0) && (frames != (unsigned)(-1)))		// TODO - this is an ugly hardcoded hack
		throw NotImplementedException("Scope is not flat!");

	Metadata::EpochTypeID vartype = Metadata::EpochType_Error;
	
	if(frames == 0)
	{
		vartype = LibJITContext.CurrentScope->GetVariableTypeByIndex(index);
	}
	else
	{
		const ScopeDescription* scope = LibJITContext.CurrentScope;
		while(scope->ParentScope)
			scope = scope->ParentScope;

		vartype = scope->GetVariableTypeByIndex(index);
		index += 0xf0000000;
	}

	if(Builder.GetInsertBlock()->getParent()->isVarArg() && (LibJITContext.CurrentScope->GetVariableOrigin(index) != VARIABLE_ORIGIN_RETURN))
	{
		VAArgInst* ptr = Builder.CreateVAArg(LibJITContext.VarArgList, Generator.GetLLVMType(vartype));
		LibJITContext.ValuesOnStack.push(ptr);
		LibJITContext.VariableMap[index] = ptr;
	}
	else
	{
		Value* ptr = LibJITContext.VariableMap[index];
		LibJITContext.ValuesOnStack.push(ptr);
	}

	TypeAnnotations.push(Metadata::MakeNonReferenceType(vartype));
}

//
// The Epoch IR uses this instruction to dereference a ref on the stack
// into a value which replaces it on the stack. We do this entire process
// at JIT time using the virtual stack and thereby eliminate a load/store
// pair on the machine stack.
//
void FunctionJITHelper::ReadRef(size_t&)
{
	LoadInst* derefvalue = Builder.CreateLoad(LibJITContext.ValuesOnStack.top());
	derefvalue->setAlignment(1);
	LibJITContext.ValuesOnStack.pop();
	LibJITContext.ValuesOnStack.push(derefvalue);
	TypeAnnotations.pop();
}

//
// When invoking type matchers, we often need to place a value and its type
// annotation on to the virtual stack in order to pass along that information
// to the dispatch routine. This instruction is used to accomplish that.
//
void FunctionJITHelper::ReadRefAnnotated(size_t&)
{
	std::vector<Value*> gepindices;
	gepindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 0));
	gepindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 0));
	Value* annotationgep = Builder.CreateGEP(LibJITContext.ValuesOnStack.top(), gepindices);

	std::vector<Value*> derefindices;
	derefindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 0));
	derefindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 1));
	Value* derefgep = Builder.CreateLoad(Builder.CreateGEP(LibJITContext.ValuesOnStack.top(), derefindices));

	Value* annotationvalue = Builder.CreateLoad(annotationgep);
	LibJITContext.ValuesOnStack.pop();
	LibJITContext.ValuesOnStack.push(derefgep);
	LibJITContext.ValuesOnStack.push(annotationvalue);
	TypeAnnotations.pop();
}

//
// The Epoch IR pops a reference and a value off the Epoch stack
// and uses it to perform a store; we can eliminate a couple of
// steps here using the virtual stack and emit just a direct store.
//
void FunctionJITHelper::Assign(size_t&)
{
	Value* reftarget = LibJITContext.ValuesOnStack.top();
	LibJITContext.ValuesOnStack.pop();
	Value* v = LibJITContext.ValuesOnStack.top();
	LibJITContext.ValuesOnStack.pop();

	if(reftarget->getType() == v->getType()->getPointerTo()->getPointerTo())
		reftarget = Builder.CreateLoad(reftarget);
	else if(reftarget->getType() == v->getType())
		v = Builder.CreateLoad(v);

	Builder.CreateStore(v, reftarget)->setAlignment(1);
}

//
// Assign a value into a reference from the virtual stack, but do
// the additional housekeeping necessary to tag the value of a sum
// typed variable as having the appropriate type.
//
void FunctionJITHelper::AssignSumType(size_t&)
{
	Value* reftarget = LibJITContext.ValuesOnStack.top();
	LibJITContext.ValuesOnStack.pop();
	Value* actualtype = LibJITContext.ValuesOnStack.top();
	LibJITContext.ValuesOnStack.pop();

	Value* typeholder = Builder.CreatePointerCast(reftarget, Type::getInt32PtrTy(Context));
	if(actualtype->getType()->getNumContainedTypes() == 1)
	{
		LoadInst* load = dyn_cast<LoadInst>(actualtype);

		std::vector<Value*> gepindices;
		gepindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 0));
		gepindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 0));
		Builder.CreateStore(Builder.CreateLoad(Builder.CreateGEP(load->getOperand(0), gepindices)), typeholder);

		std::vector<Value*> payloadgepindices;
		payloadgepindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 0));
		payloadgepindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 1));
		Value* payloadgep = Builder.CreateGEP(load->getOperand(0), payloadgepindices);
		Value* payload = Builder.CreateLoad(payloadgep);

		Value* target = Builder.CreateGEP(Builder.CreatePointerCast(reftarget, Type::getInt8PtrTy(Context)), ConstantInt::get(Type::getInt32Ty(Context), sizeof(Metadata::EpochTypeID)));
		Value* casttarget = Builder.CreatePointerCast(target, payload->getType()->getPointerTo());
		Builder.CreateStore(payload, casttarget);
	}
	else if(actualtype->getType()->getNumContainedTypes() == 2)
	{
		Builder.CreateStore(actualtype, reftarget);
	}
	else
	{
		Builder.CreateStore(actualtype, typeholder);

		Value* target = Builder.CreateGEP(Builder.CreatePointerCast(reftarget, Type::getInt8PtrTy(Context)), ConstantInt::get(Type::getInt32Ty(Context), sizeof(Metadata::EpochTypeID)));
		Value* casttarget = Builder.CreatePointerCast(target, LibJITContext.ValuesOnStack.top()->getType()->getPointerTo());
		Builder.CreateStore(LibJITContext.ValuesOnStack.top(), casttarget);

		LibJITContext.ValuesOnStack.pop();
	}
}

//
// Variant of assigning to a sum-typed variable that is used
// exclusively during construction. Since the circumstances
// are a little different, we can simplify the code a touch.
//
void FunctionJITHelper::ConstructSumType(size_t&)
{
	Value* vartype = LibJITContext.ValuesOnStack.top();
	LibJITContext.ValuesOnStack.pop();


	if(vartype->getType()->getNumContainedTypes() == 2)			// TODO - hackish?
	{
		Value* targetid = LibJITContext.ValuesOnStack.top();
		LibJITContext.ValuesOnStack.pop();

		ConstantInt* cint = dyn_cast<ConstantInt>(targetid);
		size_t vartarget = static_cast<size_t>(cint->getValue().getLimitedValue());

		Value* storagetarget = LibJITContext.VariableMap[LibJITContext.NameToIndexMap[vartarget]];

		Builder.CreateStore(vartype, storagetarget);

		return;
	}


	Value* value = LibJITContext.ValuesOnStack.top();
	LibJITContext.ValuesOnStack.pop();

	Value* targetid = LibJITContext.ValuesOnStack.top();
	LibJITContext.ValuesOnStack.pop();


	bool typeisnothing = false;
	ConstantInt* ctype = dyn_cast<ConstantInt>(vartype);
	if(ctype != NULL)
	{
		if(ctype->getValue().getLimitedValue() == Metadata::EpochType_Nothing)
			typeisnothing = true;
	}

	ConstantInt* cint = dyn_cast<ConstantInt>(targetid);
	size_t vartarget = static_cast<size_t>(cint->getValue().getLimitedValue());

	Value* storagetarget = LibJITContext.VariableMap[LibJITContext.NameToIndexMap[vartarget]];

	if(!typeisnothing)
	{
		// Set contents
		std::vector<Value*> memberindices;
		memberindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 0));
		memberindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 1));
		Value* valueholder = Builder.CreatePointerCast(Builder.CreateGEP(storagetarget, memberindices), value->getType()->getPointerTo());
		Builder.CreateStore(value, valueholder);
	}

	// Set type annotation
	{
		std::vector<Value*> memberindices;
		memberindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 0));
		memberindices.push_back(ConstantInt::get(Type::getInt32Ty(Context), 0));
		Value* typeholder = Builder.CreateGEP(storagetarget, memberindices);
		Builder.CreateStore(vartype, typeholder);
	}
}

//
// Epoch's semantics for function return values need a little bit
// of tweaking to apply to the format generally supported by LLVM.
// This routine handles the conversion process.
//
void FunctionJITHelper::SetRetValue(size_t& offset)
{
	size_t index = Fetch<size_t>(Bytecode, offset);
	Builder.CreateStore(Builder.CreateLoad(LibJITContext.VariableMap[index]), LibJITContext.InnerRetVal);
}

//
// Handle the Epoch function return instruction. We do not convert
// this directly into an LLVM return instruction because the function
// may need to do some epilog work prior to actually returning to the
// caller on the machine level; garbage collection, deterministic
// destruction, and so on are handled in this manner.
//
void FunctionJITHelper::Return(size_t&)
{
	Builder.CreateBr(LibJITContext.InnerExitBlock);
}

//
// Utility instruction for halting program execution.
//
void FunctionJITHelper::Halt(size_t&)
{
	Builder.CreateCall(Generator.Data->BuiltInFunctions[JITFunc_Runtime_Halt]);
	Builder.CreateUnreachable();
}

//
// Push a literal immediate value from the Epoch bytecode stream
// onto the virtual stack so it can be operated on later.
//
void FunctionJITHelper::Push(size_t& offset)
{
	Metadata::EpochTypeID type = Fetch<Metadata::EpochTypeID>(Bytecode, offset);
	Value* valueval;

	switch(type)
	{
	case Metadata::EpochType_Integer:
		{
			Integer32 value = Fetch<Integer32>(Bytecode, offset);
			valueval = ConstantInt::get(Type::getInt32Ty(Context), value);
		}
		break;

	case Metadata::EpochType_Identifier:
	case Metadata::EpochType_String:
		{
			StringHandle value = Fetch<StringHandle>(Bytecode, offset);
			valueval = ConstantInt::get(Type::getInt32Ty(Context), value);
		}
		break;

	case Metadata::EpochType_Real:
		{
			Real32 value = Fetch<Real32>(Bytecode, offset);
			valueval = ConstantFP::get(Type::getFloatTy(Context), value);
		}
		break;

	case Metadata::EpochType_Boolean:
		{
			bool value = Fetch<bool>(Bytecode, offset);
			valueval = ConstantInt::get(Type::getInt1Ty(Context), value);
		}
		break;

	case Metadata::EpochType_Integer16:
		{
			Integer16 value = Fetch<Integer16>(Bytecode, offset);
			valueval = ConstantInt::get(Type::getInt16Ty(Context), value);
		}
		break;

	case Metadata::EpochTypeFamily_Function:		// We only emit generic family, not the actual signature type. This is kind of hacky.
		{
			StringHandle funcname = Fetch<StringHandle>(Bytecode, offset);
			size_t offset = Generator.ExecContext.GetFunctionInstructionOffsetNoThrow(funcname);
			
			if(offset)
				valueval = Generator.GetGeneratedFunction(funcname, offset);
			else
			{
				std::map<StringHandle, const char*>::const_iterator libiter = Generator.ExecContext.JITHelpers.LibraryExports.find(funcname);
				if(libiter == Generator.ExecContext.JITHelpers.LibraryExports.end())
				{
					if(!LibJITContext.CurrentScope->HasVariable(funcname))
						throw NotImplementedException("Invalid higher order function target");

					size_t ignored = 0;
					size_t index = LibJITContext.CurrentScope->FindVariable(funcname, ignored);
					valueval = Builder.CreateLoad(LibJITContext.VariableMap[index]);
				}
				else
				{
					FunctionType* ftype = Generator.GetLLVMFunctionTypeFromSignature(funcname);
					Function* func = Generator.LibraryFunctionCache[libiter->second];

					if(!func)
						func = Generator.LibraryFunctionCache[libiter->second] = Function::Create(ftype, Function::ExternalLinkage, libiter->second, Generator.Data->CurrentModule);

					valueval = func;
				}
			}
		}
		break;

	case Metadata::EpochType_Buffer:
	default:
		throw FatalException("Unsupported type for JIT compilation");
	}

	LibJITContext.ValuesOnStack.push(valueval);
}

//
// Pop the virtual stack; this is useful for eliminating return
// values that have been ignored in the source program.
//
void FunctionJITHelper::Pop(size_t& offset)
{
	Fetch<size_t>(Bytecode, offset);
	LibJITContext.ValuesOnStack.pop();
}

//
// Shell out to the runtime infrastructure to allocate some structure-typed memory
//
void FunctionJITHelper::AllocStructure(size_t& offset)
{
	++AllocCount;

	Metadata::EpochTypeID type = Fetch<Metadata::EpochTypeID>(Bytecode, offset);
	Value* typeconst = ConstantInt::get(Type::getInt32Ty(Context), type);
	Value* handle = Builder.CreateCall(Generator.Data->BuiltInFunctions[JITFunc_Runtime_AllocStruct], typeconst);
	LibJITContext.ValuesOnStack.push(handle);

	HackStructType = type;

	// TODO - more hack
	LibJITContext.VarArgList = CreateAllocaInternal(Type::getInt8PtrTy(Context));
	Value* castlist = Builder.CreatePointerCast(LibJITContext.VarArgList, Type::getInt8PtrTy(Context));
	Builder.CreateCall(Generator.Data->BuiltInFunctions[JITFunc_Intrinsic_VAStart], castlist);
	//Builder.GetInsertBlock()->getParent()->addAttribute(0xffffffff, Attribute::NoInline);
}

//
// Copy a value into a structure object's memory space
//
void FunctionJITHelper::CopyToStructure(size_t& offset)
{
	StringHandle variablename = Fetch<StringHandle>(Bytecode, offset);
	StringHandle actualmember = Fetch<StringHandle>(Bytecode, offset);

	Value* v = LibJITContext.VariableMap[LibJITContext.NameToIndexMap[variablename]];
	Value* structptr = Builder.CreateLoad(v);

	const StructureDefinition& def = Generator.ExecContext.GetStructureDefinition(HackStructType);
	size_t memberindex = def.FindMember(actualmember);
	size_t memberoffset = def.GetMemberOffset(memberindex);
	Metadata::EpochTypeID membertype = def.GetMemberType(memberindex);

	Value* memberptr = Builder.CreateGEP(structptr, ConstantInt::get(Type::getInt32Ty(Context), memberoffset));

	Type* llvmtype = Generator.GetLLVMType(membertype);
	if(!Metadata::IsReferenceType(membertype))
		llvmtype = llvmtype->getPointerTo();

	Value* castmemberptr = Builder.CreatePointerCast(memberptr, llvmtype);

	StoreInst* store = Builder.CreateStore(LibJITContext.ValuesOnStack.top(), castmemberptr);
	store->setAlignment(1);

	LibJITContext.ValuesOnStack.pop();
}

//
// Request a deep copy of a structure object from the runtime
//
void FunctionJITHelper::CopyStructure(size_t&)
{
	++AllocCount;

	Value* structureptr = LibJITContext.ValuesOnStack.top();
	LibJITContext.ValuesOnStack.pop();
	Value* copyptr = Builder.CreateCall(Generator.Data->BuiltInFunctions[JITFunc_Runtime_CopyStruct], structureptr);
	Value* castptr = Builder.CreatePointerCast(copyptr, Type::getInt8PtrTy(Context));
	LibJITContext.ValuesOnStack.push(castptr);
}

//
// Bind reference to a structure member variable
//
void FunctionJITHelper::BindMemberRef(size_t& offset)
{
	Metadata::EpochTypeID membertype = Fetch<Metadata::EpochTypeID>(Bytecode, offset);
	size_t memberoffset = Fetch<size_t>(Bytecode, offset);

	Value* voidstructptr = LibJITContext.ValuesOnStack.top();
	if(voidstructptr->getType() == Type::getInt8PtrTy(Context)->getPointerTo())
		voidstructptr = Builder.CreateLoad(voidstructptr);
	else if(voidstructptr->getType() == Type::getInt8PtrTy(Context)->getPointerTo()->getPointerTo())
		voidstructptr = Builder.CreateLoad(Builder.CreateLoad(voidstructptr));
	Value* bytestructptr = Builder.CreatePointerCast(voidstructptr, Type::getInt8PtrTy(Context));
	Value* voidmemberptr = Builder.CreateGEP(bytestructptr, ConstantInt::get(Type::getInt32Ty(Context), memberoffset));

	Type* llvmtype = Generator.GetLLVMType(membertype);
	if(!Metadata::IsReferenceType(membertype))
		llvmtype = llvmtype->getPointerTo();

	Value* memberptr = Builder.CreatePointerCast(voidmemberptr, llvmtype);

	LibJITContext.ValuesOnStack.pop();
	LibJITContext.ValuesOnStack.push(memberptr);
	TypeAnnotations.pop();
	TypeAnnotations.push(membertype);
}

//
// Handle the trivial case of invoking library code from an Epoch program
//
void FunctionJITHelper::Invoke(size_t& offset)
{
	StringHandle target = Fetch<StringHandle>(Bytecode, offset);
	std::map<StringHandle, JIT::JITHelper>::const_iterator iter = Generator.ExecContext.JITHelpers.InvokeHelpers.find(target);
	if(iter != Generator.ExecContext.JITHelpers.InvokeHelpers.end())
	{
		if(iter->second(LibJITContext, true))
			return;
	}

	std::map<StringHandle, const char*>::const_iterator libiter = Generator.ExecContext.JITHelpers.LibraryExports.find(target);
	if(libiter == Generator.ExecContext.JITHelpers.LibraryExports.end())
		throw FatalException("Cannot invoke this function, no native code support!");

	FunctionType* ftype = Generator.GetLLVMFunctionTypeFromSignature(target);
	Function* func = Generator.LibraryFunctionCache[libiter->second];

	if(!func)
	{
		func = Generator.Data->CurrentModule->getFunction(libiter->second);
		if(!func)
			func = Generator.LibraryFunctionCache[libiter->second] = Function::Create(ftype, Function::ExternalLinkage, libiter->second, Generator.Data->CurrentModule);
	}

	const FunctionSignature& sig = Generator.ExecContext.LibraryFunctionSignatures.find(target)->second;

	std::vector<Value*> args;
	for(size_t i = sig.GetNumParameters(); i-- > 0; )
	{
		if(!sig.GetParameter(i).HasPayload)
			args.push_back(LibJITContext.ValuesOnStack.top());

		LibJITContext.ValuesOnStack.pop();
	}

	Value* v = Builder.CreateCall(func, args);

	if(func->getReturnType() != Type::getVoidTy(Generator.Data->Context))
		LibJITContext.ValuesOnStack.push(v);
}

//
// Handle the edge case of invoking a JITted type matcher from an Epoch program
//
void FunctionJITHelper::InvokeOffset(size_t& offset)
{
	StringHandle functionname = Fetch<StringHandle>(Bytecode, offset);
	size_t internaloffset = Fetch<size_t>(Bytecode, offset);

	Function* nativematcher = NULL;
	size_t numparams = 0;
	std::vector<Value*> matchervarargs;

	// Handle vanilla pattern matchers
	if(Generator.ExecContext.PatternMatcherParamCount.find(functionname) != Generator.ExecContext.PatternMatcherParamCount.end())
	{
		nativematcher = Generator.GetGeneratedPatternMatcher(functionname, internaloffset);
		numparams = Generator.ExecContext.PatternMatcherParamCount.find(functionname)->second;

		for(size_t i = 0; i < numparams; ++i)
		{
			Value* v = LibJITContext.ValuesOnStack.top();
			LibJITContext.ValuesOnStack.pop();

			matchervarargs.push_back(v);
		}
	}
	else
	{
		if(Generator.ExecContext.TypeMatcherParamCount.find(functionname) == Generator.ExecContext.TypeMatcherParamCount.end())
			throw FatalException("Cannot invoke this code");

		nativematcher = Generator.GetGeneratedTypeMatcher(functionname, internaloffset);
		numparams = Generator.ExecContext.TypeMatcherParamCount.find(functionname)->second;

		for(size_t i = 0; i < numparams; ++i)
		{
			Value* v1 = LibJITContext.ValuesOnStack.top();
			LibJITContext.ValuesOnStack.pop();

			Value* v2 = LibJITContext.ValuesOnStack.top();
			LibJITContext.ValuesOnStack.pop();

			bool isfunctionpointerty = false;
			if(v2->getType()->isPointerTy() && v2->getType()->getContainedType(0)->getNumContainedTypes() > 0)
				isfunctionpointerty = (NULL != dyn_cast<FunctionType>(v2->getType()->getContainedType(0)));

			if(v2->getType()->isPointerTy() && !isfunctionpointerty)
			{
				Metadata::EpochTypeID paramepochtype = TypeAnnotations.top();
				Value* annotation = ConstantInt::get(Type::getInt32Ty(Context), paramepochtype);
				TypeAnnotations.pop();

				matchervarargs.push_back(annotation);
				matchervarargs.push_back(Builder.CreatePointerCast(v2, Type::getInt8PtrTy(Context)));
			}
			else
			{
				LoadInst* load = dyn_cast<LoadInst>(v2);
				if(load)
				{
					matchervarargs.push_back(v1);
					matchervarargs.push_back(Builder.CreatePointerCast(load->getOperand(0), Type::getInt8PtrTy(Context)));
				}
				else
				{
					Value* stacktemp = CreateAllocaInternal(v2->getType(), true);
					Builder.CreateStore(v2, stacktemp);

					//if(v2->getType()->isPointerTy() && !isfunctionpointerty)
					//	stacktemp = Builder.CreateLoad(stacktemp);

					matchervarargs.push_back(v1);
					matchervarargs.push_back(Builder.CreatePointerCast(stacktemp, Type::getInt8PtrTy(Context)));
				}
			}
		}
	}

	CallInst* v = Builder.CreateCall(nativematcher, matchervarargs);
	//v->setIsNoInline();
	if(v->getType() != Type::getVoidTy(Context))
		LibJITContext.ValuesOnStack.push(v);
}

//
// Handle the common case of Epoch function calls from one native routine to another
//
void FunctionJITHelper::InvokeNative(size_t& offset)
{
	StringHandle target = Fetch<StringHandle>(Bytecode, offset);
	Fetch<size_t>(Bytecode, offset);		// skip dummy offset
	std::map<StringHandle, JIT::JITHelper>::const_iterator iter = Generator.ExecContext.JITHelpers.InvokeHelpers.find(target);
	if(iter != Generator.ExecContext.JITHelpers.InvokeHelpers.end())
		iter->second(LibJITContext, true);
	else
	{
		Function* targetfunc = Generator.GetGeneratedFunction(target, Generator.ExecContext.GetFunctionInstructionOffsetNoThrow(target));

		size_t paramcount = 0;
		
		const ScopeDescription& desc = Generator.ExecContext.GetScopeDescription(target);
		for(size_t i = 0; i < desc.GetVariableCount(); ++i)
		{
			if(desc.GetVariableOrigin(i) == VARIABLE_ORIGIN_PARAMETER)
			{
				Metadata::EpochTypeID vartype = desc.GetVariableTypeByIndex(i);
				++paramcount;

				// Add type signature param if need be
				if(Metadata::GetTypeFamily(vartype) == Metadata::EpochTypeFamily_SumType)
				{
					if(targetfunc->isVarArg())
						++paramcount;
				}
			}
		}

		std::vector<Value*> targetargs;
		for(size_t i = 0; i < paramcount; ++i)
		{
			Value* p = LibJITContext.ValuesOnStack.top();
			LibJITContext.ValuesOnStack.pop();
			targetargs.push_back(p);
		}
		std::reverse(targetargs.begin(), targetargs.end());

		CallInst* v = Builder.CreateCall(targetfunc, targetargs);

		//if(Generator.ExecContext.AutoGeneratedConstructors.find(target) != Generator.ExecContext.AutoGeneratedConstructors.end())
		//	v->setIsNoInline();

		if(v->getType() != Type::getVoidTy(Context))
			LibJITContext.ValuesOnStack.push(v);
	}
}

//
// Generate LLVM bitcode for indirect function invocation (i.e. function pointer)
//
void FunctionJITHelper::InvokeIndirect(size_t& offset)
{
	StringHandle funcnameholder = Fetch<StringHandle>(Bytecode, offset);
	Value* func = Builder.CreateLoad(LibJITContext.VariableMap[LibJITContext.NameToIndexMap[funcnameholder]]);

	FunctionType* ftype = dyn_cast<FunctionType>(func->getType()->getContainedType(0));

	std::vector<Value*> args;
	for(size_t i = 0; i < ftype->getNumParams(); ++i)
	{
		args.push_back(LibJITContext.ValuesOnStack.top());
		LibJITContext.ValuesOnStack.pop();
	}

	std::reverse(args.begin(), args.end());

	if(ftype->getReturnType() != Type::getVoidTy(Context))
		LibJITContext.ValuesOnStack.push(Builder.CreateCall(func, args));
	else
		Builder.CreateCall(func, args);
}

//
// Generate type matching dispatcher prolog code for a given Epoch function set
//
void FunctionJITHelper::TypeMatch(size_t& offset)
{
	offset = EndOffset;
}

void FunctionJITHelper::PatternMatch(size_t& offset)
{
	offset = EndOffset;
}

void FunctionJITHelper::CopyBuffer(size_t&)
{
	++AllocCount;

	Value* buffer = LibJITContext.ValuesOnStack.top();
	LibJITContext.ValuesOnStack.pop();
	
	Value* clone = Builder.CreateCall(Generator.Data->BuiltInFunctions[JITFunc_Runtime_CopyBuffer], buffer);
	LibJITContext.ValuesOnStack.push(clone);
}

//
// Generate native code for type matching/dispatch when called from
// a parent routine which is already JIT compiled.
//
void NativeCodeGenerator::AddNativeTypeMatcher(size_t beginoffset, size_t endoffset)
{
	Function* matcherfunction = Builder.GetInsertBlock()->getParent();
	//matcherfunction->addAttribute(0xffffffff, Attribute::NoInline);

	std::vector<Value*> reftypes;
	std::vector<Value*> reftargets;

	std::vector<Value*> parampayloadptrs;
	std::vector<Value*> providedtypeholders;

	unsigned typematchindex = 0;
	StringHandle entityname = 0;
	bool needholders = true;

	bool usegc = false;

	for(size_t offset = beginoffset; offset <= endoffset; )
	{
		Bytecode::Instruction instruction = Bytecode[offset++];
		switch(instruction)
		{
		// Need to handle this so we can skip through the byte stream correctly
		case Bytecode::Instructions::BeginEntity:
			entityname = Fetch<Integer32>(Bytecode, offset);
			Fetch<StringHandle>(Bytecode, offset);
			break;

		// Ignore these for now
		case Bytecode::Instructions::EndEntity:
		case Bytecode::Instructions::Halt:
			break;

		case Bytecode::Instructions::TypeMatch:
			{
				StringHandle targetentity = Fetch<StringHandle>(Bytecode, offset);
				Fetch<size_t>(Bytecode, offset);
				size_t paramcount = Fetch<size_t>(Bytecode, offset);

				Function::arg_iterator argiter = matcherfunction->arg_begin();
				for(size_t i = 0; i < paramcount; ++i)
				{
					Fetch<bool>(Bytecode, offset);
					Fetch<Metadata::EpochTypeID>(Bytecode, offset);
				}

				if(ExecContext.SuppressGC.find(targetentity) == ExecContext.SuppressGC.end())
					usegc = true;
			}
			break;

		default:
			throw FatalException("Invalid opcode in native type matcher");
		}
	}

	if(!usegc)
		ExecContext.SuppressGC.insert(entityname);


	// The purpose of this loop is to hoist all stack allocations from the inner
	// type matching basic blocks out to the entry block. This allows the LLVM
	// optimizer to convert the allocas to registers, which eliminates dynamic
	// stack resizing during the type match process - a solid performance win.
	for(size_t offset = beginoffset; offset <= endoffset; )
	{
		Bytecode::Instruction instruction = Bytecode[offset++];
		switch(instruction)
		{
		// Need to handle this so we can skip through the byte stream correctly
		case Bytecode::Instructions::BeginEntity:
			entityname = Fetch<Integer32>(Bytecode, offset);
			Fetch<StringHandle>(Bytecode, offset);

			if(ExecContext.SuppressGC.find(entityname) == ExecContext.SuppressGC.end())
				matcherfunction->setGC("EpochGC");

			break;

		// Ignore these for now
		case Bytecode::Instructions::EndEntity:
		case Bytecode::Instructions::Halt:
			break;

		case Bytecode::Instructions::TypeMatch:
			{
				Fetch<StringHandle>(Bytecode, offset);
				Fetch<size_t>(Bytecode, offset);
				size_t paramcount = Fetch<size_t>(Bytecode, offset);

				Function::arg_iterator argiter = matcherfunction->arg_begin();
				for(size_t i = 0; i < paramcount; ++i)
				{
					Fetch<bool>(Bytecode, offset);
					Fetch<Metadata::EpochTypeID>(Bytecode, offset);
					parampayloadptrs.push_back(Builder.CreateAlloca(Type::getInt8PtrTy(Data->Context)));
					providedtypeholders.push_back(Builder.CreateAlloca(Type::getInt32Ty(Data->Context)));

					if(matcherfunction->hasGC())
					{
						{
							Value* signature = ConstantInt::get(Type::getInt32Ty(Data->Context), 0xffffffff);
							Value* constant = Builder.CreateIntToPtr(signature, Type::getInt8PtrTy(Data->Context));
							Value* castptr = Builder.CreatePointerCast(parampayloadptrs.back(), Type::getInt8PtrTy(Data->Context)->getPointerTo());
							Builder.CreateCall2(Data->BuiltInFunctions[JITFunc_Intrinsic_GCRoot], castptr, constant);
						}
						{
							Value* signature = ConstantInt::get(Type::getInt32Ty(Data->Context), 0xfffffffe);
							Value* constant = Builder.CreateIntToPtr(signature, Type::getInt8PtrTy(Data->Context));
							Value* castptr = Builder.CreatePointerCast(providedtypeholders.back(), Type::getInt8PtrTy(Data->Context)->getPointerTo());
							Builder.CreateCall2(Data->BuiltInFunctions[JITFunc_Intrinsic_GCRoot], castptr, constant);
						}
					}
				}
			}
			break;

		default:
			throw FatalException("Invalid opcode in native type matcher");
		}
	}

	for(size_t offset = beginoffset; offset <= endoffset; )
	{
		Bytecode::Instruction instruction = Bytecode[offset++];
		switch(instruction)
		{
		// Need to handle this so we can skip through the byte stream correctly
		case Bytecode::Instructions::BeginEntity:
			Fetch<Integer32>(Bytecode, offset);
			Fetch<StringHandle>(Bytecode, offset);
			break;

		// Ignore these for now
		case Bytecode::Instructions::EndEntity:
		case Bytecode::Instructions::Halt:
			break;

		case Bytecode::Instructions::TypeMatch:
			{
				Fetch<StringHandle>(Bytecode, offset);
				Fetch<size_t>(Bytecode, offset);
				size_t paramcount = Fetch<size_t>(Bytecode, offset);

				Function::arg_iterator argiter = matcherfunction->arg_begin();
				for(size_t i = 0; i < paramcount; ++i)
				{
					bool expectref = Fetch<bool>(Bytecode, offset);
					Metadata::EpochTypeID expecttype = Fetch<Metadata::EpochTypeID>(Bytecode, offset);
					expectref |= Metadata::IsReferenceType(expecttype);

					if(needholders)
					{
						BasicBlock* nextparamblock = BasicBlock::Create(Data->Context, "nextparam", matcherfunction);

						Value* reftype = argiter;
						++argiter;
						Value* reftarget = argiter;
						++argiter;

						reftypes.push_back(reftype);
						reftargets.push_back(reftarget);

						Builder.CreateStore(reftypes[i], providedtypeholders[i]);
						Builder.CreateStore(reftargets[i], parampayloadptrs[i]);

						BasicBlock* handlesumtypeblock = BasicBlock::Create(Data->Context, "handlesumtype", matcherfunction);

						Value* providedtypefamily = Builder.CreateAnd(Builder.CreateLoad(providedtypeholders[i]), 0x7f000000);
						Value* issumtype = Builder.CreateICmpEQ(providedtypefamily, ConstantInt::get(Type::getInt32Ty(Data->Context), Metadata::EpochTypeFamily_SumType));
						Builder.CreateCondBr(issumtype, handlesumtypeblock, nextparamblock);

						Builder.SetInsertPoint(handlesumtypeblock);

						Value* rt = Builder.CreateLoad(Builder.CreatePointerCast(reftargets[i], Type::getInt32PtrTy(Data->Context)));
						Builder.CreateStore(rt, providedtypeholders[i]);

						if(Metadata::GetTypeFamily(expecttype) == Metadata::EpochTypeFamily_SumType)
						{
							Builder.CreateStore(reftargets[i], parampayloadptrs[i]);
						}
						else
						{
							Value* gep = Builder.CreateGEP(reftargets[i], ConstantInt::get(Type::getInt32Ty(Data->Context), sizeof(Metadata::EpochTypeID)));
							Builder.CreateStore(gep, parampayloadptrs[i]);
						}
						Builder.CreateBr(nextparamblock);
						Builder.SetInsertPoint(nextparamblock);
					}
				}

				needholders = false;
			}
			break;

		default:
			throw FatalException("Invalid opcode in native type matcher");
		}
	}


	for(size_t offset = beginoffset; offset <= endoffset; )
	{
		Bytecode::Instruction instruction = Bytecode[offset++];
		switch(instruction)
		{
		case Bytecode::Instructions::BeginEntity:
			Fetch<Integer32>(Bytecode, offset);
			entityname = Fetch<StringHandle>(Bytecode, offset);
			break;

		case Bytecode::Instructions::EndEntity:
			break;

		case Bytecode::Instructions::Halt:
			Builder.CreateCall(Data->BuiltInFunctions[JITFunc_Runtime_HaltExt], ConstantInt::get(Type::getInt32Ty(Data->Context), entityname));
			Builder.CreateUnreachable();
			break;

		case Bytecode::Instructions::TypeMatch:
			{
				StringHandle funcname = Fetch<StringHandle>(Bytecode, offset);
				size_t matchoffset = Fetch<size_t>(Bytecode, offset);
				size_t paramcount = Fetch<size_t>(Bytecode, offset);

				std::vector<Value*> actualparams;
				BasicBlock* nexttypematcher = BasicBlock::Create(Data->Context, "nexttypematcher", matcherfunction);

				for(size_t i = 0; i < paramcount; ++i)
				{
					bool expectref = Fetch<bool>(Bytecode, offset);
					Metadata::EpochTypeID expecttype = Fetch<Metadata::EpochTypeID>(Bytecode, offset);
					expectref |= Metadata::IsReferenceType(expecttype);
					expecttype = Metadata::MakeNonReferenceType(expecttype);

					Value* nonref = Builder.CreateAnd(Builder.CreateLoad(providedtypeholders[i]), 0x7fffffff);
					Value* nomatch = Builder.CreateICmpNE(nonref, ConstantInt::get(Type::getInt32Ty(Data->Context), expecttype));
					Value* notexpectsumtype = ConstantInt::get(Type::getInt1Ty(Data->Context), Metadata::GetTypeFamily(expecttype) != Metadata::EpochTypeFamily_SumType);

					BasicBlock* nextparamblock = BasicBlock::Create(Data->Context, "nextparam", matcherfunction);

					Value* bailflag = Builder.CreateAnd(nomatch, notexpectsumtype);
					Builder.CreateCondBr(bailflag, nexttypematcher, nextparamblock);

					Builder.SetInsertPoint(nextparamblock);

					if(expecttype != Metadata::EpochType_Nothing)
					{
						Type* llvmtype = GetLLVMType(expecttype)->getPointerTo()->getPointerTo();
						Value* v = Builder.CreatePointerCast(parampayloadptrs[i], llvmtype);
						v = Builder.CreateLoad(v);

						if(!expectref)
							v = Builder.CreateLoad(v);

						actualparams.push_back(v);
					}
					else
					{
						actualparams.push_back(ConstantInt::get(Type::getInt32Ty(Data->Context), 0xbaadf00d));
					}
				}

				std::vector<Value*> resolvedargs;
				for(std::vector<Value*>::const_iterator argiter = actualparams.begin(); argiter != actualparams.end(); ++argiter)
					resolvedargs.push_back(*argiter);
				std::reverse(resolvedargs.begin(), resolvedargs.end());


				Function* targetinnerfunc = GetGeneratedFunction(funcname, matchoffset);

				if(targetinnerfunc->getReturnType() != Type::getVoidTy(Data->Context))
				{
					CallInst* typematchret = Builder.CreateCall(targetinnerfunc, resolvedargs);
					Builder.CreateRet(typematchret);
				}
				else
				{
					Builder.CreateCall(targetinnerfunc, resolvedargs);
					Builder.CreateRetVoid();
				}

				//BasicBlock* bookmark = Builder.GetInsertBlock();

				//size_t foooffset = ExecContext.EntityOffsetMap[funcname];
				//jithelper.DoTypeMatchedFunction(foooffset, ExecContext.GetEntityEndOffset(foooffset), funcname, resolvedargs, matcherfunction);

				//Builder.SetInsertPoint(bookmark);
				//Builder.CreateBr(jithelper.LibJITContext.InnerEntryBlock);

				Builder.SetInsertPoint(nexttypematcher);
			}

			++typematchindex;
			break;

		default:
			throw FatalException("Invalid instruction in type matcher");
		}
	}
}



void NativeCodeGenerator::AddNativePatternMatcher(size_t beginoffset, size_t endoffset)
{
	Function* matcherfunction = Builder.GetInsertBlock()->getParent();
	std::vector<Value*> argholders;

	unsigned matchindex = 0;
	StringHandle entityname = 0;

	for(size_t offset = beginoffset; offset <= endoffset; )
	{
		Bytecode::Instruction instruction = Bytecode[offset++];
		switch(instruction)
		{
		case Bytecode::Instructions::BeginEntity:
			Fetch<Integer32>(Bytecode, offset);
			entityname = Fetch<StringHandle>(Bytecode, offset);
			break;

		case Bytecode::Instructions::EndEntity:
			break;

		case Bytecode::Instructions::Halt:
			Builder.CreateCall(Data->BuiltInFunctions[JITFunc_Runtime_Halt]);
			Builder.CreateUnreachable();
			break;

		case Bytecode::Instructions::PatternMatch:
			{
				StringHandle targetfunc = Fetch<StringHandle>(Bytecode, offset);
				size_t internaloffset = Fetch<size_t>(Bytecode, offset);
				size_t paramcount = Fetch<size_t>(Bytecode, offset);

				std::vector<Value*> actualparams;
				BasicBlock* nextmatcher = BasicBlock::Create(Data->Context, "nextmatcher", matcherfunction);

				if(argholders.empty())
				{
					Function::arg_iterator argiter = matcherfunction->arg_begin();

					for(size_t i = 0; i < paramcount; ++i)
					{
						Value* argument = argiter;
						++argiter;

						argholders.push_back(argument);
					}
				}

				for(size_t i = 0; i < paramcount; ++i)
				{
					Metadata::EpochTypeID paramtype = Fetch<Metadata::EpochTypeID>(Bytecode, offset);
					Byte needsmatch = Fetch<Byte>(Bytecode, offset);

					BasicBlock* nextparamblock = BasicBlock::Create(Data->Context, "nextparam", matcherfunction);
					
					if(needsmatch)
					{
						Value* eq = NULL;

						switch(paramtype)
						{
						case Metadata::EpochType_Integer:
							{
								Integer32 value = Fetch<Integer32>(Bytecode, offset);
								Value* constval = ConstantInt::get(Type::getInt32Ty(Data->Context), value);
								eq = Builder.CreateICmpEQ(constval, argholders[i]);
							}
							break;

						default:
							throw NotImplementedException("Cannot JIT pattern matcher of this type");
						}

						Builder.CreateCondBr(eq, nextparamblock, nextmatcher);
					}
					else
					{
						actualparams.push_back(argholders[i]);
						Builder.CreateBr(nextparamblock);						
					}

					Builder.SetInsertPoint(nextparamblock);
				}

				Function* targetinnerfunc = GetGeneratedFunction(targetfunc, internaloffset);

				std::vector<Value*> resolvedargs;
				for(std::vector<Value*>::const_reverse_iterator argiter = actualparams.rbegin(); argiter != actualparams.rend(); ++argiter)
					resolvedargs.push_back(*argiter);

				if(targetinnerfunc->getReturnType() != Type::getVoidTy(Data->Context))
				{
					CallInst* matchret = Builder.CreateCall(targetinnerfunc, resolvedargs);
					Builder.CreateRet(matchret);
				}
				else
				{
					Builder.CreateCall(targetinnerfunc, resolvedargs);
					Builder.CreateRetVoid();
				}

				Builder.SetInsertPoint(nextmatcher);
			}

			++matchindex;
			break;

		default:
			throw FatalException("Invalid instruction in pattern matcher");
		}
	}
}


Value* NativeCodeGenerator::MarshalArgument(Value* arg, Metadata::EpochTypeID type)
{
	type = Metadata::MakeNonReferenceType(type);
	switch(type)
	{
	case Metadata::EpochType_Boolean:
		return Builder.CreateCast(Instruction::SExt, arg, Type::getInt32Ty(Data->Context));

	case Metadata::EpochType_Buffer:
		return Builder.CreateCall(Data->BuiltInFunctions[JITFunc_Runtime_GetBuffer], arg);

	case Metadata::EpochType_Integer:
	case Metadata::EpochType_Integer16:
	case Metadata::EpochType_Real:
		return arg;

	case Metadata::EpochType_String:
		return Builder.CreateCall(Data->BuiltInFunctions[JITFunc_Runtime_GetString], arg);
	}

	if(Metadata::IsStructureType(type))
		return Builder.CreateCall(Data->BuiltInFunctions[JITFunc_Marshal_ConvertStructure], arg);

	Metadata::EpochTypeFamily family = Metadata::GetTypeFamily(type);
	if(family == Metadata::EpochTypeFamily_Function)
		return Builder.CreateCall(Data->BuiltInFunctions[JITFunc_Marshal_GenCallback], Builder.CreatePointerCast(arg, Type::getInt8PtrTy(Data->Context)));

	throw NotImplementedException("Cannot marshal parameter of this type to an external function");
}

Value* NativeCodeGenerator::MarshalArgumentReverse(Value* arg, Metadata::EpochTypeID type)
{
	type = Metadata::MakeNonReferenceType(type);
	switch(type)
	{
	case Metadata::EpochType_Boolean:
		return Builder.CreateCast(Instruction::Trunc, arg, Type::getInt1Ty(Data->Context));

	case Metadata::EpochType_Integer:
	case Metadata::EpochType_Integer16:
	case Metadata::EpochType_Real:
		return arg;

	case Metadata::EpochType_String:
		return Builder.CreateCall(Data->BuiltInFunctions[JITFunc_Runtime_PoolString], arg);
	}

	throw NotImplementedException("Cannot marshal parameter of this type to a callback function");
}

Value* NativeCodeGenerator::MarshalReturn(Value* ret, Metadata::EpochTypeID type)
{
	switch(type)
	{
	case Metadata::EpochType_Boolean:
		return Builder.CreateCast(Instruction::Trunc, ret, Type::getInt1Ty(Data->Context));

	case Metadata::EpochType_Buffer:
		// TODO
		break;

	case Metadata::EpochType_Integer:
	case Metadata::EpochType_Integer16:
	case Metadata::EpochType_Real:
		return ret;

	case Metadata::EpochType_String:
		return Builder.CreateCall(Data->BuiltInFunctions[JITFunc_Runtime_PoolString], ret);
	}

	throw NotImplementedException("Cannot marshal parameter of this type to an external function");
}

Value* NativeCodeGenerator::MarshalReturnReverse(Value* ret, Metadata::EpochTypeID type)
{
	switch(type)
	{
	case Metadata::EpochType_Boolean:
	case Metadata::EpochType_Integer:
	case Metadata::EpochType_Integer16:
	case Metadata::EpochType_Real:
		return ret;

	case Metadata::EpochType_Buffer:
		return Builder.CreateCall(Data->BuiltInFunctions[JITFunc_Runtime_GetBuffer], ret);
	}

	throw NotImplementedException("Cannot marshal return value of this type from a callback function");
}

void NativeCodeGenerator::MarshalReferencePostCall(Value* ref, Value* fixuptarget, Metadata::EpochTypeID type)
{
	type = Metadata::MakeNonReferenceType(type);
	switch(type)
	{
	case Metadata::EpochType_Boolean:
	case Metadata::EpochType_Integer:
	case Metadata::EpochType_Integer16:
	case Metadata::EpochType_Real:
	case Metadata::EpochType_String:
	case Metadata::EpochType_Buffer:
		return;
	}

	if(Metadata::IsStructureType(type))
	{
		Builder.CreateCall2(Data->BuiltInFunctions[JITFunc_Marshal_FixupStructure], ref, Builder.CreateLoad(fixuptarget));
		return;
	}

	throw NotImplementedException("Cannot marshal reference of this type back from an external function");
}

void NativeCodeGenerator::MarshalCleanup(Value* val, Metadata::EpochTypeID type)
{
	type = Metadata::MakeNonReferenceType(type);
	switch(type)
	{
	case Metadata::EpochType_Boolean:
	case Metadata::EpochType_Integer:
	case Metadata::EpochType_Integer16:
	case Metadata::EpochType_Real:
	case Metadata::EpochType_Buffer:
	case Metadata::EpochType_String:
		return;
	}

	if(Metadata::IsStructureType(type))
	{
		Builder.CreateCall(Data->BuiltInFunctions[JITFunc_Marshal_Cleanup], val);
		return;
	}

	Metadata::EpochTypeFamily family = Metadata::GetTypeFamily(type);
	if(family == Metadata::EpochTypeFamily_Function)
		return;

	throw NotImplementedException("Cannot clean up result of marshaling this type back from an external function");
}

void* NativeCodeGenerator::GenerateCallbackWrapper(void* targetfunc)
{
	// TODO - cache callback wrappers!

	Function* llvmtargetfunc = reinterpret_cast<Function*>(ExecContext.GeneratedJITFunctionCode[targetfunc].first);

	StringHandle alias = ExecContext.GeneratedJITFunctionCode[targetfunc].second;
	FunctionType* ftype = GetExternalFunctionType(alias);
	Function* wrapfunc = Function::Create(ftype, GlobalValue::ExternalWeakLinkage, "", Data->CurrentModule);

	// TODO - support non-stdcall callbacks?
	wrapfunc->setCallingConv(CallingConv::X86_StdCall);

	BasicBlock* block = BasicBlock::Create(Data->Context, "wrap", wrapfunc);
	Builder.SetInsertPoint(block);

	//Builder.CreateCall(Data->BuiltInFunctions[JITFunc_Runtime_GCBookmark]);

	std::vector<Value*> args;
	Function::arg_iterator argiter = wrapfunc->arg_begin();
	const ScopeDescription& desc = ExecContext.GetScopeDescription(alias);
	for(size_t i = 0; i < desc.GetVariableCount(); ++i)
	{
		while(i < desc.GetVariableCount() && desc.GetVariableOrigin(i) != VARIABLE_ORIGIN_PARAMETER)
			++i;

		if(i >= desc.GetVariableCount())
			break;

		Value* val = (Argument*)(argiter);
		val = MarshalArgumentReverse(val, desc.GetVariableTypeByIndex(i));

		args.push_back(val);
		++argiter;
	}

	Constant* constaddr = ConstantInt::get(Type::getInt32Ty(Data->Context), (Integer32)(targetfunc));
	Value* jittedtarget = Builder.CreateIntToPtr(constaddr, llvmtargetfunc->getType());

	if(wrapfunc->getFunctionType()->getReturnType() == Type::getVoidTy(Data->Context))
	{
		Builder.CreateCall(jittedtarget, args);
		//Builder.CreateCall(Data->BuiltInFunctions[JITFunc_Runtime_GCUnbookmark]);
		Builder.CreateRetVoid();
	}
	else
	{
		Value* retval = MarshalReturnReverse(Builder.CreateCall(jittedtarget, args), desc.GetReturnVariableType());
		//Builder.CreateCall(Data->BuiltInFunctions[JITFunc_Runtime_GCUnbookmark]);
		Builder.CreateRet(retval);
	}

	//Data->CurrentModule->dump();	

	return LazyEngine->getPointerToFunction(wrapfunc);
}



void JIT::DestructLLVMModule()
{
	EpochGC::ClearGCContextInfo();
}

void JIT::DestructLLVMNativeCode()
{
	delete LazyEngine;
	LazyEngine = NULL;
}


void NativeCodeGenerator::NotifyFunctionEmitted(const llvm::Function& function, void* code, size_t size, const EmittedFunctionDetails&)
{
	EpochGC::SetGCFunctionBounds(&function, code, size);

	llvm::Function* f = const_cast<llvm::Function*>(&function);
	ExecContext.GeneratedJITFunctionCode[code] = std::make_pair(f, Data->GeneratedFunctionToNameMap[f]);

	//if(Data->GeneratedFunctionToNameMap[f])
	//	std::wcout << L"Emitted function " << ExecContext.GetPooledString(Data->GeneratedFunctionToNameMap[f]) << L" at address " << code << L" - " << (void*)((char*)(code) + size) << std::endl;
}

